def f(args):
def x(*y):
print(y)
grp = GreenPool()
list(grp.starmap(x, [[1, 2, 3] for x in range(1000)]))
grp.waitall()
def run(self, action_ref, parameters=None, count=10, concurrency=None):
if not concurrency:
concurrency = count
pool = GreenPool(concurrency)
client = Client()
execution_ids = []
def schedule_action(action_ref, parameters):
execution = LiveAction()
execution.action = action_ref
execution.parameters = parameters
execution = client.liveactions.create(execution)
execution_ids.append(execution.id)
start_timestamp = time.time()
for index in range(0, count):
pool.spawn(schedule_action, action_ref, parameters)
pool.waitall()
end_timestamp = time.time()
delta = (end_timestamp - start_timestamp)
print('Scheduled %s action executions in %ss.' % (count, delta))
def tests(status, test):
pool = GreenPool(size=500)
for host, s in status['servers'].iteritems():
for t in test:
if t.name in s:
pool.spawn_n(t.test, host, s)
pool.waitall()
def tests(status, test):
pool = GreenPool(size=500)
for host, s in status['servers'].iteritems():
for t in test:
if t.name in s:
pool.spawn_n(t.test, host, s)
pool.waitall()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = []
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(
_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.expiring_objects_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
containers_to_delete.append(container)
for o in self.swift.iter_objects(self.expiring_objects_account,
container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' %
(container, obj)).hexdigest(), 16)
if obj_process % processes != process:
continue
timestamp, actual_obj = obj.split('-', 1)
timestamp = int(timestamp)
if timestamp > int(time()):
break
pool.spawn_n(self.delete_object, actual_obj, timestamp,
container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %s %s') %
(container, str(err)))
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = set([])
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug('Run begin')
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(
_('Pass beginning; '
'%(containers)s possible containers; '
'%(objects)s possible objects') % {
'containers': containers,
'objects': objects
})
for container, obj in self.iter_cont_objs_to_expire():
containers_to_delete.add(container)
if not obj:
continue
timestamp, actual_obj = obj.split('-', 1)
timestamp = int(timestamp)
if timestamp > int(time()):
break
pool.spawn_n(self.delete_object, actual_obj, timestamp,
container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %(container)s '
'%(err)s') % {
'container': container,
'err': str(err)
})
self.logger.debug('Run end')
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = []
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.expiring_objects_account):
container = c['name']
timestamp = int(container)
if timestamp > int(time()):
break
containers_to_delete.append(container)
for o in self.swift.iter_objects(self.expiring_objects_account,
container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
timestamp, actual_obj = obj.split('-', 1)
timestamp = int(timestamp)
if timestamp > int(time()):
break
pool.spawn_n(
self.delete_object, actual_obj, timestamp,
container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %s %s') %
(container, str(err)))
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def run_once(self, *args, **kwargs):
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
self.report_containers = 0
containers_to_delete = []
try:
self.logger.debug(_('Run begin'))
containers, objects = \
self.swift.get_account_info(self.sample_account)
self.logger.info(_('Pass beginning; %s possible containers; %s '
'possible objects') % (containers, objects))
for c in self.swift.iter_containers(self.sample_account):
container = c['name']
try:
timestamp, account = container.split('_', 1)
timestamp = float(timestamp)
except ValueError:
self.logger.debug('ValueError: %s, '
'need more than 1 value to unpack' % \
container)
else:
if processes > 0:
obj_proc = int(hashlib.md5(container).hexdigest(), 16)
if obj_proc % processes != process:
continue
n = (float(time()) // self.sample_rate) * self.sample_rate
if timestamp <= n:
containers_to_delete.append(container)
pool.spawn_n(self.aggregate_container, container)
pool.waitall()
for container in containers_to_delete:
try:
self.logger.debug('delete container: %s' % container)
self.swift.delete_container(self.sample_account, container,
acceptable_statuses=(
2, HTTP_NOT_FOUND,
HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %s %s') %
(container, str(err)))
tenants_to_fillup = list()
for c in self.swift.iter_containers(self.aggregate_account):
tenant_id = c['name']
if processes > 0:
c_proc = int(hashlib.md5(tenant_id).hexdigest(), 16)
if c_proc % processes != process:
continue
tenants_to_fillup.append(tenant_id)
# fillup lossed usage data
self.fillup_lossed_usage_data(tenants_to_fillup)
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def test_high_client_load():
eventlet.spawn_n(fake_service, "tcp://127.0.0.1:6805")
clients = GreenPool()
for i in xrange(0, 100):
clients.spawn(fake_client, "tcp://127.0.0.1:6804",
"%s:%s" % (os.getpid(), i))
clients.waitall()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug('Run begin')
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(
_('Pass beginning; '
'%(containers)s possible containers; '
'%(objects)s possible objects') % {
'containers': containers,
'objects': objects
})
task_containers = list(self.iter_task_containers_to_expire())
# delete_task_iter is a generator to yield a dict of
# task_container, task_object, delete_timestamp, target_path
# to handle delete actual object and pop the task from the queue.
delete_task_iter = self.round_robin_order(
self.iter_task_to_expire(task_containers))
for delete_task in delete_task_iter:
pool.spawn_n(self.delete_object, **delete_task)
pool.waitall()
for container in task_containers:
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %(container)s '
'%(err)s') % {
'container': container,
'err': str(err)
})
self.logger.debug('Run end')
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
containers_to_delete = set([])
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug('Run begin')
containers, objects = \
self.swift.get_account_info(self.expiring_objects_account)
self.logger.info(_('Pass beginning; '
'%(containers)s possible containers; '
'%(objects)s possible objects') % {
'containers': containers, 'objects': objects})
for container, obj in self.iter_cont_objs_to_expire():
containers_to_delete.add(container)
if not obj:
continue
timestamp, actual_obj = obj.split('-', 1)
timestamp = int(timestamp)
if timestamp > int(time()):
break
pool.spawn_n(
self.delete_object, actual_obj, timestamp,
container, obj)
pool.waitall()
for container in containers_to_delete:
try:
self.swift.delete_container(
self.expiring_objects_account,
container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %(container)s '
'%(err)s') % {'container': container,
'err': str(err)})
self.logger.debug('Run end')
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
def test_high_workload():
# fire up three services to receive in roundrobin style, giving
# each an ident so we can make sure they're working that way
eventlet.spawn_n(fake_service, "tcp://127.0.0.1:6900", 1)
eventlet.spawn_n(fake_service, "tcp://127.0.0.1:6900", 2)
eventlet.spawn_n(fake_service, "tcp://127.0.0.1:6900", 3)
clients = GreenPool()
# fire up a bunch of clients to thrash it at random
for i in xrange(0, 100):
clients.spawn(fake_client, "tcp://127.0.0.1:6802", "%s:%s" % (os.getpid(), i))
clients.waitall()
def run(self, count=100):
pool = GreenPool(count)
client = Client()
rule_ids = []
name_patterns = ['key1', 'key2', 'key3', 'key4', 'key5']
def create_rule(rule):
try:
rule = client.rules.create(rule)
except Exception as e:
# Rule already exists, ignore the error
print(e)
return
rule_ids.append(rule.id)
start_timestamp = time.time()
index_name_pattern = 0
for index in range(0, count):
rule = Rule()
rule.name = 'rule_%s' % (index)
rule.pack = 'default'
rule.trigger = {'type': 'core.st2.key_value_pair.create'}
# Use uniform distribution of names so if COUNT is 100, each key
# will be used COUNT / len(KEYS)
if index_name_pattern >= len(name_patterns):
index_name_pattern = 0
pattern = name_patterns[index_name_pattern]
rule.criteria = {
'trigger.object.name': {
'pattern': (pattern),
'type': 'equals'
}
}
rule.action = {'ref': 'core.noop'}
index_name_pattern += 1
pool.spawn(create_rule, rule)
pool.waitall()
end_timestamp = time.time()
delta = (end_timestamp - start_timestamp)
print('Created %d rules in %ss.' % (count, delta))
def test_high_client_load():
test_context = {'clients': 0, 'services': 0}
pool = GreenPool()
pool.spawn(fake_service,
"tcp://127.0.0.1:6801", test_context)
for i in xrange(0, 10):
pool.spawn(fake_client, "tcp://127.0.0.1:6800",
"%s" % i, test_context)
pool.waitall()
assert_equal(test_context['clients'], 10)
assert_equal(test_context['services'], 100)
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
processes, process = self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug(_('Run begin'))
for o in self.swift.iter_objects(self.restoring_object_account,
self.todo_container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (self.todo_container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
pool.spawn_n(self.start_object_restoring, obj)
pool.waitall()
for o in self.swift.iter_objects(self.restoring_object_account,
self.restoring_container):
obj = o['name'].encode('utf8')
if processes > 0:
obj_process = int(
hashlib.md5('%s/%s' % (self.restoring_container, obj)).
hexdigest(), 16)
if obj_process % processes != process:
continue
pool.spawn_n(self.check_object_restored, obj)
pool.waitall()
self.logger.debug(_('Run end'))
self.report(final=True)
except (Exception, Timeout) as e:
report_exception(self.logger, _('Unhandled exception'), self.client)
def discovery(status, test):
pool = GreenPool(size=500)
for d in settings.discovery:
servers = d().get_servers() # [('ip', 'host')]
for server in servers:
ip = server[0]
host = server[1]
if host in settings.exclude:
continue
if host not in status["servers"]: # do discovery
status["servers"][host] = {}
logging.info("performing discovery on %r", server)
for t in test:
pool.spawn_n(t.discover, ip, status["servers"][host])
status["servers"][host]["ip"] = ip
pool.waitall()
def discovery(status, test):
pool = GreenPool(size=500)
for d in settings.discovery:
servers = d().get_servers() # [('ip', 'host')]
for server in servers:
ip = server[0]
host = server[1]
if host in settings.exclude:
continue
if host not in status['servers']: # do discovery
status['servers'][host] = {}
logging.info('performing discovery on %r', server)
for t in test:
pool.spawn_n(t.discover, ip, status['servers'][host])
status['servers'][host]['ip'] = ip
pool.waitall()
def imap(requests, prefetch=True, size=2):
"""Concurrently converts a generator object of Requests to
a generator of Responses.
:param requests: a generator of Request objects.
:param prefetch: If False, the content will not be downloaded immediately.
:param size: Specifies the number of requests to make at a time. default is 2
"""
def send(r):
r.send(prefetch)
return r.response
pool = GreenPool(size)
for r in pool.imap(send, requests):
yield r
pool.waitall()
def map(requests, prefetch=True, size=None):
"""Concurrently converts a list of Requests to Responses.
:param requests: a collection of Request objects.
:param prefetch: If False, the content will not be downloaded immediately.
:param size: Specifies the number of requests to make at a time. If None, no throttling occurs.
"""
requests = list(requests)
pool = GreenPool(size) if size else None
jobs = [send(r, pool, prefetch=prefetch) for r in requests]
if pool is not None:
pool.waitall()
else:
[j.wait() for j in jobs]
return [r.response for r in requests]
def imap(requests, stream=False, size=2):
"""Concurrently converts a generator object of Requests to
a generator of Responses.
:param requests: a generator of Request objects.
:param stream: If True, the content will not be downloaded immediately.
:param size: Specifies the number of requests to make at a time. default is 2
"""
pool = GreenPool(size)
def send(r):
return r.send(stream=stream)
for r in pool.imap_unordered(send, requests):
yield r
pool.waitall()
def get_all_highest_druing_previous_days(days, result):
stock_codes = get_stock_codes()
gp = GreenPool()
today = datetime.now().strftime("%Y-%m-%d")
filename = "%s_highest_%s_days.json" % (today, days)
if os.path.exists(filename):
with open(filename) as f:
result.update(json.load(f))
return
for stock in stock_codes:
gp.spawn(
eventlet_handle,
functools.partial(get_edge_during_previous_days,
coloumn='high',
days=days,
highest=True), stock, result)
gp.waitall()
with open(filename, 'w') as f:
json.dump(result, f, indent=2)
def main(myid, queue, concurrency, delay=5.0, duration=DURATION):
counter = 0
created = list()
results = LightQueue(concurrency * 10)
pool = GreenPool(concurrency)
api = AccountClient({'namespace': NS}, pool_maxsize=concurrency+1)
now = start = checkpoint = time.time()
pool.starmap(create_loop, [(api, 'buck-%d-%d' % (myid, n), results)
for n in range(concurrency)])
while now - start < duration:
try:
res = results.get(timeout=delay)
created.append(res)
counter += 1
except Empty:
pass
if now - checkpoint > delay:
print("Proc %d: %d updates in %fs, %f updates per second." % (
myid, counter, now - checkpoint,
counter / (now - checkpoint)))
counter = 0
checkpoint = now
now = time.time()
for coro in pool.coroutines_running:
coro.kill()
while not results.empty():
created.append(results.get(block=False))
end = time.time()
rate = len(created) / (end - start)
print("Proc %d: end. %d updates in %fs, %f updates per second." % (
myid, len(created), end - start, rate))
time.sleep(2)
print("Proc %d: cleaning..." % myid)
del_req = {'dtime': time.time()}
# Do not delete twice (or an exception is raised)
uniq_ct = set(created)
for _ in pool.starmap(api.container_update,
[(ACCOUNT, n, del_req) for n in uniq_ct]):
pass
pool.waitall()
queue.put(rate)
return 0
class ServiceContainer(object):
def __init__(self, service_cls, worker_ctx_cls, config):
self.service_cls = service_cls
self.worker_ctx_cls = worker_ctx_cls
self.service_name = get_service_name(service_cls)
self.config = config
self.max_workers = config.get(MAX_WORKERS_KEY) or DEFAULT_MAX_WORKERS
self.dependencies = DependencySet()
for dep in prepare_dependencies(self):
self.dependencies.add(dep)
self.started = False
self._worker_pool = GreenPool(size=self.max_workers)
self._active_threads = set()
self._protected_threads = set()
self._being_killed = False
self._died = Event()
@property
def entrypoints(self):
return filter(is_entrypoint_provider, self.dependencies)
@property
def injections(self):
return filter(is_injection_provider, self.dependencies)
def start(self):
""" Start a container by starting all the dependency providers.
"""
_log.debug('starting %s', self)
self.started = True
with log_time(_log.debug, 'started %s in %0.3f sec', self):
self.dependencies.all.prepare()
self.dependencies.all.start()
def stop(self):
""" Stop the container gracefully.
First all entrypoints are asked to ``stop()``.
This ensures that no new worker threads are started.
It is the providers' responsibility to gracefully shut down when
``stop()`` is called on them and only return when they have stopped.
After all entrypoints have stopped the container waits for any
active workers to complete.
After all active workers have stopped the container stops all
injections.
At this point there should be no more managed threads. In case there
are any managed threads, they are killed by the container.
"""
if self._died.ready():
_log.debug('already stopped %s', self)
return
_log.debug('stopping %s', self)
with log_time(_log.debug, 'stopped %s in %0.3f sec', self):
dependencies = self.dependencies
# entrypoint deps have to be stopped before injection deps
# to ensure that running workers can successfully complete
dependencies.entrypoints.all.stop()
# there might still be some running workers, which we have to
# wait for to complete before we can stop injection dependencies
self._worker_pool.waitall()
# it should be safe now to stop any injection as there is no
# active worker which could be using it
dependencies.injections.all.stop()
# finally, stop nested dependencies
dependencies.nested.all.stop()
# just in case there was a provider not taking care of its workers,
# or a dependency not taking care of its protected threads
self._kill_active_threads()
self._kill_protected_threads()
self.started = False
self._died.send(None)
def kill(self, exc):
""" Kill the container in a semi-graceful way.
All non-protected managed threads are killed first. This includes
all active workers generated by :meth:`ServiceContainer.spawn_worker`.
Next, dependencies are killed. Finally, any remaining protected threads
are killed.
The container dies with the given ``exc``.
"""
if self._being_killed:
# this happens if a managed thread exits with an exception
# while the container is being killed or another caller
# behaves in a similar manner
_log.debug('already killing %s ... waiting for death', self)
self._died.wait()
self._being_killed = True
if self._died.ready():
_log.debug('already stopped %s', self)
return
_log.info('killing %s due to "%s"', self, exc)
self.dependencies.entrypoints.all.kill(exc)
self._kill_active_threads()
self.dependencies.all.kill(exc)
self._kill_protected_threads()
self.started = False
self._died.send_exception(exc)
def wait(self):
""" Block until the container has been stopped.
If the container was stopped using ``kill(exc)``,
``wait()`` raises ``exc``.
Any unhandled exception raised in a managed thread or in the
life-cycle management code also causes the container to be
``kill()``ed, which causes an exception to be raised from ``wait()``.
"""
return self._died.wait()
def spawn_worker(self, provider, args, kwargs,
context_data=None, handle_result=None):
""" Spawn a worker thread for running the service method decorated
with an entrypoint ``provider``.
``args`` and ``kwargs`` are used as arguments for the service
method.
``context_data`` is used to initialize a ``WorkerContext``.
``handle_result`` is an optional callback which may be passed
in by the calling entrypoint provider. It is called with the
result returned or error raised by the service method.
"""
service = self.service_cls()
worker_ctx = self.worker_ctx_cls(
self, service, provider.name, args, kwargs, data=context_data)
_log.debug('spawning %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
gt = self._worker_pool.spawn(self._run_worker, worker_ctx,
handle_result)
self._active_threads.add(gt)
gt.link(self._handle_thread_exited)
return worker_ctx
def spawn_managed_thread(self, run_method, protected=False):
""" Spawn a managed thread to run ``run_method``.
Threads can be marked as ``protected``, which means the container will
not forcibly kill them until after all dependencies have been killed.
Dependencies that require a managed thread to complete their kill
procedure should ensure to mark them as ``protected``.
Any uncaught errors inside ``run_method`` cause the container to be
killed.
It is the caller's responsibility to terminate their spawned threads.
Threads are killed automatically if they are still running after
all dependencies are stopped during :meth:`ServiceContainer.stop`.
Entrypoints may only create separate threads using this method,
to ensure they are life-cycle managed.
"""
gt = eventlet.spawn(run_method)
if not protected:
self._active_threads.add(gt)
else:
self._protected_threads.add(gt)
gt.link(self._handle_thread_exited)
return gt
def _run_worker(self, worker_ctx, handle_result):
_log.debug('setting up %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
if not worker_ctx.parent_call_stack:
_log.debug('starting call chain',
extra=worker_ctx.extra_for_logging)
_log.debug('call stack for %s: %s',
worker_ctx, '->'.join(worker_ctx.call_id_stack),
extra=worker_ctx.extra_for_logging)
with log_time(_log.debug, 'ran worker %s in %0.3fsec', worker_ctx):
self.dependencies.injections.all.inject(worker_ctx)
self.dependencies.all.worker_setup(worker_ctx)
result = exc = None
try:
_log.debug('calling handler for %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
method = getattr(worker_ctx.service, worker_ctx.method_name)
with log_time(_log.debug, 'ran handler for %s in %0.3fsec',
worker_ctx):
result = method(*worker_ctx.args, **worker_ctx.kwargs)
except Exception as e:
log_worker_exception(worker_ctx, e)
exc = e
with log_time(_log.debug, 'tore down worker %s in %0.3fsec',
worker_ctx):
_log.debug('signalling result for %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
self.dependencies.injections.all.worker_result(
worker_ctx, result, exc)
_log.debug('tearing down %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
self.dependencies.all.worker_teardown(worker_ctx)
self.dependencies.injections.all.release(worker_ctx)
if handle_result is not None:
_log.debug('handling result for %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
with log_time(_log.debug, 'handled result for %s in %0.3fsec',
worker_ctx):
handle_result(worker_ctx, result, exc)
def _kill_active_threads(self):
""" Kill all managed threads that were not marked as "protected" when
they were spawned.
This set will include all worker threads generated by
:meth:`ServiceContainer.spawn_worker`.
See :meth:`ServiceContainer.spawn_managed_thread`
"""
num_active_threads = len(self._active_threads)
if num_active_threads:
_log.warning('killing %s active thread(s)', num_active_threads)
for gt in list(self._active_threads):
gt.kill()
def _kill_protected_threads(self):
""" Kill any managed threads marked as protected when they were
spawned.
See :meth:`ServiceContainer.spawn_managed_thread`
"""
num_protected_threads = len(self._protected_threads)
if num_protected_threads:
_log.warning('killing %s protected thread(s)',
num_protected_threads)
for gt in list(self._protected_threads):
gt.kill()
def _handle_thread_exited(self, gt):
self._active_threads.discard(gt)
self._protected_threads.discard(gt)
try:
gt.wait()
except greenlet.GreenletExit:
# we don't care much about threads killed by the container
# this can happen in stop() and kill() if providers
# don't properly take care of their threads
_log.warning('%s thread killed by container', self)
except Exception as exc:
_log.error('%s thread exited with error', self,
exc_info=True)
# any error raised inside an active thread is unexpected behavior
# and probably a bug in the providers or container
# to be safe we kill the container
self.kill(exc)
def __str__(self):
return '<ServiceContainer [{}] at 0x{:x}>'.format(
self.service_name, id(self))
def entrance():
pool = GreenPool(100)
for x in range(10):
pool.spawn(fetcher)
pool.waitall()
class ServiceContainer(object):
def __init__(self, service_cls, worker_ctx_cls, config):
self.service_cls = service_cls
self.worker_ctx_cls = worker_ctx_cls
self.service_name = get_service_name(service_cls)
self.config = config
self.max_workers = (config.get(MAX_WORKERS_CONFIG_KEY)
or DEFAULT_MAX_WORKERS)
self.dependencies = DependencySet()
for dep in prepare_dependencies(self):
self.dependencies.add(dep)
self.started = False
self._worker_pool = GreenPool(size=self.max_workers)
self._active_threads = {}
self._protected_threads = set()
self._being_killed = False
self._died = Event()
@property
def entrypoints(self):
return filter(is_entrypoint_provider, self.dependencies)
@property
def injections(self):
return filter(is_injection_provider, self.dependencies)
def start(self):
""" Start a container by starting all the dependency providers.
"""
_log.debug('starting %s', self)
self.started = True
with _log_time('started %s', self):
self.dependencies.all.prepare()
self.dependencies.all.start()
def stop(self):
""" Stop the container gracefully.
First all entrypoints are asked to ``stop()``.
This ensures that no new worker threads are started.
It is the providers' responsibility to gracefully shut down when
``stop()`` is called on them and only return when they have stopped.
After all entrypoints have stopped the container waits for any
active workers to complete.
After all active workers have stopped the container stops all
injections.
At this point there should be no more managed threads. In case there
are any managed threads, they are killed by the container.
"""
if self._died.ready():
_log.debug('already stopped %s', self)
return
if self._being_killed:
# this race condition can happen when a container is hosted by a
# runner and yields during its kill method; if it's unlucky in
# scheduling the runner will try to stop() it before self._died
# has a result
_log.debug('already being killed %s', self)
try:
self._died.wait()
except:
pass # don't re-raise if we died with an exception
return
_log.debug('stopping %s', self)
with _log_time('stopped %s', self):
dependencies = self.dependencies
# entrypoint deps have to be stopped before injection deps
# to ensure that running workers can successfully complete
dependencies.entrypoints.all.stop()
# there might still be some running workers, which we have to
# wait for to complete before we can stop injection dependencies
self._worker_pool.waitall()
# it should be safe now to stop any injection as there is no
# active worker which could be using it
dependencies.injections.all.stop()
# finally, stop nested dependencies
dependencies.nested.all.stop()
# just in case there was a provider not taking care of its workers,
# or a dependency not taking care of its protected threads
self._kill_active_threads()
self._kill_protected_threads()
self.started = False
self._died.send(None)
def kill(self, exc_info=None):
""" Kill the container in a semi-graceful way.
All non-protected managed threads are killed first. This includes
all active workers generated by :meth:`ServiceContainer.spawn_worker`.
Next, dependencies are killed. Finally, any remaining protected threads
are killed.
If ``exc_info`` is provided, the exception will be raised by
:meth:`~wait``.
"""
if self._being_killed:
# this happens if a managed thread exits with an exception
# while the container is being killed or if multiple errors
# happen simultaneously
_log.debug('already killing %s ... waiting for death', self)
try:
self._died.wait()
except:
pass # don't re-raise if we died with an exception
return
self._being_killed = True
if self._died.ready():
_log.debug('already stopped %s', self)
return
if exc_info is not None:
_log.info('killing %s due to %s', self, exc_info[1])
else:
_log.info('killing %s', self)
# protect against dependencies that throw during kill; the container
# is already dying with an exception, so ignore anything else
def safely_kill_dependencies(dep_set):
try:
dep_set.kill()
except Exception as exc:
_log.warning('Dependency raised `%s` during kill', exc)
safely_kill_dependencies(self.dependencies.entrypoints.all)
self._kill_active_threads()
safely_kill_dependencies(self.dependencies.all)
self._kill_protected_threads()
self.started = False
self._died.send(None, exc_info)
def wait(self):
""" Block until the container has been stopped.
If the container was stopped due to an exception, ``wait()`` will
raise it.
Any unhandled exception raised in a managed thread or in the
life-cycle management code also causes the container to be
``kill()``ed, which causes an exception to be raised from ``wait()``.
"""
return self._died.wait()
def spawn_worker(self,
provider,
args,
kwargs,
context_data=None,
handle_result=None):
""" Spawn a worker thread for running the service method decorated
with an entrypoint ``provider``.
``args`` and ``kwargs`` are used as arguments for the service
method.
``context_data`` is used to initialize a ``WorkerContext``.
``handle_result`` is an optional function which may be passed
in by the entrypoint provider. It is called with the result returned
or error raised by the service method. If provided it must return a
value for ``result`` and ``exc_info`` to propagate to dependencies;
these may be different to those returned by the service method.
"""
if self._being_killed:
_log.info("Worker spawn prevented due to being killed")
raise ContainerBeingKilled()
service = self.service_cls()
worker_ctx = self.worker_ctx_cls(self,
service,
provider,
args,
kwargs,
data=context_data)
_log.debug('spawning %s', worker_ctx)
gt = self._worker_pool.spawn(self._run_worker, worker_ctx,
handle_result)
self._active_threads[gt] = provider
gt.link(self._handle_thread_exited)
return worker_ctx
def spawn_managed_thread(self, run_method, protected=False):
""" Spawn a managed thread to run ``run_method``.
Threads can be marked as ``protected``, which means the container will
not forcibly kill them until after all dependencies have been killed.
Dependencies that require a managed thread to complete their kill
procedure should ensure to mark them as ``protected``.
Any uncaught errors inside ``run_method`` cause the container to be
killed.
It is the caller's responsibility to terminate their spawned threads.
Threads are killed automatically if they are still running after
all dependencies are stopped during :meth:`ServiceContainer.stop`.
Entrypoints may only create separate threads using this method,
to ensure they are life-cycle managed.
"""
gt = eventlet.spawn(run_method)
if not protected:
self._active_threads[gt] = MANAGED_THREAD
else:
self._protected_threads.add(gt)
gt.link(self._handle_thread_exited)
return gt
def _run_worker(self, worker_ctx, handle_result):
_log.debug('setting up %s', worker_ctx)
if not worker_ctx.parent_call_stack:
_log.debug('starting call chain')
_log.debug('call stack for %s: %s', worker_ctx,
'->'.join(worker_ctx.call_id_stack))
with _log_time('ran worker %s', worker_ctx):
self.dependencies.injections.all.inject(worker_ctx)
self.dependencies.all.worker_setup(worker_ctx)
result = exc_info = None
method = getattr(worker_ctx.service, worker_ctx.provider.name)
try:
_log.debug('calling handler for %s', worker_ctx)
with _log_time('ran handler for %s', worker_ctx):
result = method(*worker_ctx.args, **worker_ctx.kwargs)
except Exception as exc:
_log.debug('error handling worker %s: %s',
worker_ctx,
exc,
exc_info=True)
exc_info = sys.exc_info()
if handle_result is not None:
_log.debug('handling result for %s', worker_ctx)
with _log_time('handled result for %s', worker_ctx):
result, exc_info = handle_result(worker_ctx, result,
exc_info)
with _log_time('tore down worker %s', worker_ctx):
_log.debug('signalling result for %s', worker_ctx)
self.dependencies.injections.all.worker_result(
worker_ctx, result, exc_info)
# we don't need this any more, and breaking the cycle means
# this can be reclaimed immediately, rather than waiting for a
# gc sweep
del exc_info
self.dependencies.all.worker_teardown(worker_ctx)
self.dependencies.injections.all.release(worker_ctx)
def _kill_active_threads(self):
""" Kill all managed threads that were not marked as "protected" when
they were spawned.
This set will include all worker threads generated by
:meth:`ServiceContainer.spawn_worker`.
See :meth:`ServiceContainer.spawn_managed_thread`
"""
num_active_threads = len(self._active_threads)
if num_active_threads:
_log.warning('killing %s active thread(s)', num_active_threads)
for gt, provider in list(self._active_threads.items()):
if provider is not MANAGED_THREAD:
description = '{}.{}'.format(self.service_name,
provider.name)
_log.warning('killing active thread for %s', description)
gt.kill()
def _kill_protected_threads(self):
""" Kill any managed threads marked as protected when they were
spawned.
See :meth:`ServiceContainer.spawn_managed_thread`
"""
num_protected_threads = len(self._protected_threads)
if num_protected_threads:
_log.warning('killing %s protected thread(s)',
num_protected_threads)
for gt in list(self._protected_threads):
gt.kill()
def _handle_thread_exited(self, gt):
self._active_threads.pop(gt, None)
self._protected_threads.discard(gt)
try:
gt.wait()
except GreenletExit:
# we don't care much about threads killed by the container
# this can happen in stop() and kill() if providers
# don't properly take care of their threads
_log.warning('%s thread killed by container', self)
except Exception:
_log.error('%s thread exited with error', self, exc_info=True)
# any error raised inside an active thread is unexpected behavior
# and probably a bug in the providers or container.
# to be safe we call self.kill() to kill our dependencies and
# provide the exception info to be raised in self.wait().
self.kill(sys.exc_info())
def __repr__(self):
service_name = repr_safe_str(self.service_name)
return '<ServiceContainer [{}] at 0x{:x}>'.format(
service_name, id(self))
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
# This if-clause will be removed when general task queue feature is
# implemented.
if not self.dequeue_from_legacy:
self.logger.info('This node is not configured to dequeue tasks '
'from the legacy queue. This node will '
'not process any expiration tasks. At least '
'one node in your cluster must be configured '
'with dequeue_from_legacy == true.')
return
self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug('Run begin')
task_account_container_list_to_delete = list()
for task_account, my_index, divisor in \
self.iter_task_accounts_to_expire():
container_count, obj_count = \
self.swift.get_account_info(task_account)
# the task account is skipped if there are no task container
if not container_count:
continue
self.logger.info(_(
'Pass beginning for task account %(account)s; '
'%(container_count)s possible containers; '
'%(obj_count)s possible objects') % {
'account': task_account,
'container_count': container_count,
'obj_count': obj_count})
task_account_container_list = \
[(task_account, task_container) for task_container in
self.iter_task_containers_to_expire(task_account)]
task_account_container_list_to_delete.extend(
task_account_container_list)
# delete_task_iter is a generator to yield a dict of
# task_account, task_container, task_object, delete_timestamp,
# target_path to handle delete actual object and pop the task
# from the queue.
delete_task_iter = \
self.round_robin_order(self.iter_task_to_expire(
task_account_container_list, my_index, divisor))
for delete_task in delete_task_iter:
pool.spawn_n(self.delete_object, **delete_task)
pool.waitall()
for task_account, task_container in \
task_account_container_list_to_delete:
try:
self.swift.delete_container(
task_account, task_container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %(account)s '
'%(container)s %(err)s') % {
'account': task_account,
'container': task_container, 'err': str(err)})
self.logger.debug('Run end')
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
infiles = glob('SNAP_ASCII/RUN_ANALYSIS001/gen*.clones')
outfiles = glob('SNAP_ASCII/RUN_ANALYSIS002/gen*.clones')
infiles.sort()
outfiles.sort()
# ifname = file with the "ingroups" (tested for polymorphism)
# ofname = file with the "outgroups" (diff between in/out => divergance)
have_printed_header = False
MonKeyTestPath = glob('analys*/src/MonKeyTest')[0]
def process_generation(ifname,ofname):
global have_printed_header
gen_num = int(re.findall('gen(\d+)',ifname)[0])
gen_num2 = int(re.findall('gen(\d+)',ofname)[0])
assert(gen_num==gen_num2)
args = [MonKeyTestPath,'-1',ifname,'-2',ofname]
mkout = subprocess.check_output(args)
mklines = mkout.splitlines()
if not have_printed_header:
sys.stdout.write("generation\t"+mklines[0]+'\n')
have_printed_header = True
for line in mklines[1:]:
if line=='': next
sys.stdout.write("%i\t%s\n"%(gen_num,line))
gp = GreenPool(size=10)
for ifname,ofname in zip(infiles,outfiles):
process_generation(ifname,ofname)
# gp.spawn(process_generation,ifname,ofname)
gp.waitall()
class AsynchronousSection(object):
"""Allows calling function asynchronously with waiting on exit."""
MIN_POOL_SIZE = 1
def __init__(self, size=0, ignore_errors_num=0):
"""Initialises.
:param size: the max number of parallel tasks
:param ignore_errors_num:
number of errors which does not stop the execution
"""
self.executor = GreenPool(max(size, self.MIN_POOL_SIZE))
self.ignore_errors_num = ignore_errors_num
self.errors = []
self.tasks = set()
def __enter__(self):
self.errors[:] = []
return self
def __exit__(self, etype, *_):
self.wait(etype is not None)
def execute(self, func, *args, **kwargs):
"""Calls function asynchronously."""
if 0 <= self.ignore_errors_num < len(self.errors):
raise RuntimeError("Too many errors.")
gt = self.executor.spawn(func, *args, **kwargs)
self.tasks.add(gt)
gt.link(self.on_complete)
return gt
def on_complete(self, gt):
"""Callback to handle task completion."""
try:
gt.wait()
except Exception as e:
logger.error("Task failed: %s", six.text_type(e))
self.errors.append(sys.exc_info())
finally:
self.tasks.discard(gt)
def wait(self, ignore_errors=False):
"""Waits until all tasks will be completed.
Do not use directly, will be called from context manager.
"""
self.executor.waitall()
if len(self.errors) > 0:
for exc_info in self.errors:
logger.exception("error details.", exc_info=exc_info)
self.errors[:] = []
if not ignore_errors:
raise RuntimeError(
"Operations completed with errors.\n"
"See log for more details."
)
class AsynchronousSection(object):
"""Allows calling function asynchronously with waiting on exit."""
MIN_POOL_SIZE = 1
def __init__(self, size=0, ignore_errors_num=0):
"""Initialises.
:param size: the max number of parallel tasks
:param ignore_errors_num:
number of errors which does not stop the execution
"""
self.executor = GreenPool(max(size, self.MIN_POOL_SIZE))
self.ignore_errors_num = ignore_errors_num
self.errors = []
self.tasks = set()
def __enter__(self):
self.errors[:] = []
return self
def __exit__(self, etype, *_):
self.wait(etype is not None)
def execute(self, func, *args, **kwargs):
"""Calls function asynchronously."""
if 0 <= self.ignore_errors_num < len(self.errors):
raise RuntimeError("Too many errors.")
gt = self.executor.spawn(func, *args, **kwargs)
self.tasks.add(gt)
gt.link(self.on_complete)
return gt
def on_complete(self, gt):
"""Callback to handle task completion."""
try:
gt.wait()
except Exception as e:
logger.error("Task failed: %s", six.text_type(e))
self.errors.append(sys.exc_info())
finally:
self.tasks.discard(gt)
def wait(self, ignore_errors=False):
"""Waits until all tasks will be completed.
Do not use directly, will be called from context manager.
"""
self.executor.waitall()
if len(self.errors) > 0:
for exc_info in self.errors:
logger.exception("error details.", exc_info=exc_info)
self.errors[:] = []
if not ignore_errors:
raise RuntimeError(
"Operations completed with errors.\n"
"See log for more details."
)
class Service(ConsumerMixin):
def __init__(self, controllercls,
connection, exchange, topic,
pool=None, poolsize=1000):
self.nodeid = UIDGEN()
if pool is None:
self.procpool = GreenPool(size=poolsize)
else:
self.procpool = pool
self.connection = connection
self.controller = controllercls()
self.topic = topic
self.greenlet = None
self.messagesem = Semaphore()
self.consume_ready = Event()
node_topic = "{}.{}".format(self.topic, self.nodeid)
self.queues = [entities.get_topic_queue(exchange, topic),
entities.get_topic_queue(exchange, node_topic),
entities.get_fanout_queue(topic), ]
self._channel = None
self._consumers = None
def start(self):
# self.connection = newrpc.create_connection()
if self.greenlet is not None and not self.greenlet.dead:
raise RuntimeError()
self.greenlet = eventlet.spawn(self.run)
def get_consumers(self, Consumer, channel):
return [Consumer(self.queues, callbacks=[self.on_message, ]), ]
def on_consume_ready(self, connection, channel, consumers, **kwargs):
self._consumers = consumers
self._channel = channel
self.consume_ready.send(None)
def on_consume_end(self, connection, channel):
self.consume_ready.reset()
def on_message(self, body, message):
# need a semaphore to stop killing between message ack()
# and spawning process.
with self.messagesem:
self.procpool.spawn(self.handle_request, body)
message.ack()
def handle_request(self, body):
newrpc.process_message(self.connection, self.controller, body)
def wait(self):
try:
self.greenlet.wait()
except greenlet.GreenletExit:
pass
return self.procpool.waitall()
def kill(self):
if self.greenlet is not None and not self.greenlet.dead:
self.should_stop = True
#with self.messagesem:
#self.greenlet.kill()
self.greenlet.wait()
if self._consumers:
for c in self._consumers:
c.cancel()
if self._channel is not None:
self._channel.close()
def link(self, *args, **kwargs):
return self.greenlet.link(*args, **kwargs)
def kill_processes(self):
for g in self.procpool.coroutines_running:
g.kill()
import httplib2 directly, httplib2 doesn't natively support cooperative
yielding.
monkey_patch httplib2 will make httplib2 green.
'''
from eventlet.greenpool import GreenPool
import eventlet
import random
import httplib2
# uncomment this line to green httplib2
# httplib2 = eventlet.import_patched('httplib2')
def worker(url):
print "worker "+str(random.random())
h = httplib2.Http()
resp, content = h.request(url, "GET")
return resp
pool = GreenPool(size=10)
results = pool.imap(worker, open("urls.txt", 'r'))
for result in results:
print result
print "done...."
pool.waitall()
class Checker(object):
def __init__(self, namespace, concurrency=50, error_file=None):
self.pool = GreenPool(concurrency)
self.error_file = error_file
if self.error_file:
f = open(self.error_file, 'a')
self.error_writer = csv.writer(f, delimiter=' ')
conf = {'namespace': namespace}
self.account_client = AccountClient(conf)
self.container_client = ContainerClient(conf)
self.blob_client = BlobClient()
self.accounts_checked = 0
self.containers_checked = 0
self.objects_checked = 0
self.chunks_checked = 0
self.account_not_found = 0
self.container_not_found = 0
self.object_not_found = 0
self.chunk_not_found = 0
self.account_exceptions = 0
self.container_exceptions = 0
self.object_exceptions = 0
self.chunk_exceptions = 0
self.list_cache = {}
self.running = {}
def write_error(self, target):
error = [target.account]
if target.container:
error.append(target.container)
if target.obj:
error.append(target.obj)
if target.chunk:
error.append(target.chunk)
self.error_writer.writerow(error)
def check_chunk(self, target):
chunk = target.chunk
obj_listing = self.check_obj(target)
error = False
if chunk not in obj_listing:
print(' Chunk %s missing in object listing' % target)
error = True
# checksum = None
else:
# TODO check checksum match
# checksum = obj_listing[chunk]['hash']
pass
try:
self.blob_client.chunk_head(chunk)
except exc.NotFound as e:
self.chunk_not_found += 1
error = True
print(' Not found chunk "%s": %s' % (target, str(e)))
except Exception as e:
self.chunk_exceptions += 1
error = True
print(' Exception chunk "%s": %s' % (target, str(e)))
if error and self.error_file:
self.write_error(target)
self.chunks_checked += 1
def check_obj(self, target, recurse=False):
account = target.account
container = target.container
obj = target.obj
if (account, container, obj) in self.running:
self.running[(account, container, obj)].wait()
if (account, container, obj) in self.list_cache:
return self.list_cache[(account, container, obj)]
self.running[(account, container, obj)] = Event()
print('Checking object "%s"' % target)
container_listing = self.check_container(target)
error = False
if obj not in container_listing:
print(' Object %s missing in container listing' % target)
error = True
# checksum = None
else:
# TODO check checksum match
# checksum = container_listing[obj]['hash']
pass
results = []
try:
_, resp = self.container_client.content_show(acct=account,
ref=container,
path=obj)
except exc.NotFound as e:
self.object_not_found += 1
error = True
print(' Not found object "%s": %s' % (target, str(e)))
except Exception as e:
self.object_exceptions += 1
error = True
print(' Exception object "%s": %s' % (target, str(e)))
else:
results = resp
chunk_listing = dict()
for chunk in results:
chunk_listing[chunk['url']] = chunk
self.objects_checked += 1
self.list_cache[(account, container, obj)] = chunk_listing
self.running[(account, container, obj)].send(True)
del self.running[(account, container, obj)]
if recurse:
for chunk in chunk_listing:
t = target.copy()
t.chunk = chunk
self.pool.spawn_n(self.check_chunk, t)
if error and self.error_file:
self.write_error(target)
return chunk_listing
def check_container(self, target, recurse=False):
account = target.account
container = target.container
if (account, container) in self.running:
self.running[(account, container)].wait()
if (account, container) in self.list_cache:
return self.list_cache[(account, container)]
self.running[(account, container)] = Event()
print('Checking container "%s"' % target)
account_listing = self.check_account(target)
error = False
if container not in account_listing:
error = True
print(' Container %s missing in account listing' % target)
marker = None
results = []
while True:
try:
resp = self.container_client.container_list(acct=account,
ref=container,
marker=marker)
except exc.NotFound as e:
self.container_not_found += 1
error = True
print(' Not found container "%s": %s' % (target, str(e)))
break
except Exception as e:
self.container_exceptions += 1
error = True
print(' Exception container "%s": %s' % (target, str(e)))
break
if resp['objects']:
marker = resp['objects'][-1]['name']
else:
break
results.extend(resp['objects'])
container_listing = dict()
for obj in results:
container_listing[obj['name']] = obj
self.containers_checked += 1
self.list_cache[(account, container)] = container_listing
self.running[(account, container)].send(True)
del self.running[(account, container)]
if recurse:
for obj in container_listing:
t = target.copy()
t.obj = obj
self.pool.spawn_n(self.check_obj, t, True)
if error and self.error_file:
self.write_error(target)
return container_listing
def check_account(self, target, recurse=False):
account = target.account
if account in self.running:
self.running[account].wait()
if account in self.list_cache:
return self.list_cache[account]
self.running[account] = Event()
print('Checking account "%s"' % target)
error = False
marker = None
results = []
while True:
try:
resp = self.account_client.containers_list(account,
marker=marker)
except Exception as e:
self.account_exceptions += 1
error = True
print(' Exception account "%s": %s' % (target, str(e)))
break
if resp['listing']:
marker = resp['listing'][-1][0]
else:
break
results.extend(resp['listing'])
containers = dict()
for e in results:
containers[e[0]] = (e[1], e[2])
self.list_cache[account] = containers
self.running[account].send(True)
del self.running[account]
self.accounts_checked += 1
if recurse:
for container in containers:
t = target.copy()
t.container = container
self.pool.spawn_n(self.check_container, t, True)
if error and self.error_file:
self.write_error(target)
return containers
def check(self, target):
if target.chunk and target.obj and target.container:
self.pool.spawn_n(self.check_chunk, target)
elif target.obj and target.container:
self.pool.spawn_n(self.check_obj, target, True)
elif target.container:
self.pool.spawn_n(self.check_container, target, True)
else:
self.pool.spawn_n(self.check_account, target, True)
def wait(self):
self.pool.waitall()
def report(self):
def _report_stat(name, stat):
print("{0:18}: {1}".format(name, stat))
print()
print('Report')
_report_stat("Accounts checked", self.accounts_checked)
if self.account_not_found:
_report_stat("Missing accounts", self.account_not_found)
if self.account_exceptions:
_report_stat("Exceptions", self.account_not_found)
print()
_report_stat("Containers checked", self.containers_checked)
if self.container_not_found:
_report_stat("Missing containers", self.container_not_found)
if self.container_exceptions:
_report_stat("Exceptions", self.container_exceptions)
print()
_report_stat("Objects checked", self.objects_checked)
if self.object_not_found:
_report_stat("Missing objects", self.object_not_found)
if self.object_exceptions:
_report_stat("Exceptions", self.object_exceptions)
print()
_report_stat("Chunks checked", self.chunks_checked)
if self.chunk_not_found:
_report_stat("Missing chunks", self.chunk_not_found)
if self.chunk_exceptions:
_report_stat("Exceptions", self.chunk_exceptions)
class ServiceContainer(object):
def __init__(self, service_cls, config):
self.service_cls = service_cls
self.config = config
self.service_name = get_service_name(service_cls)
self.shared_extensions = {}
self.max_workers = (
config.get(MAX_WORKERS_CONFIG_KEY) or DEFAULT_MAX_WORKERS)
self.serializer, self.accept = serialization.setup(self.config)
self.entrypoints = SpawningSet()
self.dependencies = SpawningSet()
self.subextensions = SpawningSet()
for attr_name, dependency in inspect.getmembers(service_cls,
is_dependency):
bound = dependency.bind(self.interface, attr_name)
self.dependencies.add(bound)
self.subextensions.update(iter_extensions(bound))
for method_name, method in inspect.getmembers(service_cls, is_method):
entrypoints = getattr(method, ENTRYPOINT_EXTENSIONS_ATTR, [])
for entrypoint in entrypoints:
bound = entrypoint.bind(self.interface, method_name)
self.entrypoints.add(bound)
self.subextensions.update(iter_extensions(bound))
self.started = False
self._worker_pool = GreenPool(size=self.max_workers)
self._worker_threads = {}
self._managed_threads = {}
self._being_killed = False
self._died = Event()
@property
def extensions(self):
return SpawningSet(
self.entrypoints | self.dependencies | self.subextensions
)
@property
def interface(self):
""" An interface to this container for use by extensions.
"""
return self
def start(self):
""" Start a container by starting all of its extensions.
"""
_log.debug('starting %s', self)
self.started = True
with _log_time('started %s', self):
self.extensions.all.setup()
self.extensions.all.start()
def stop(self):
""" Stop the container gracefully.
First all entrypoints are asked to ``stop()``.
This ensures that no new worker threads are started.
It is the extensions' responsibility to gracefully shut down when
``stop()`` is called on them and only return when they have stopped.
After all entrypoints have stopped the container waits for any
active workers to complete.
After all active workers have stopped the container stops all
dependency providers.
At this point there should be no more managed threads. In case there
are any managed threads, they are killed by the container.
"""
if self._died.ready():
_log.debug('already stopped %s', self)
return
if self._being_killed:
# this race condition can happen when a container is hosted by a
# runner and yields during its kill method; if it's unlucky in
# scheduling the runner will try to stop() it before self._died
# has a result
_log.debug('already being killed %s', self)
try:
self._died.wait()
except:
pass # don't re-raise if we died with an exception
return
_log.debug('stopping %s', self)
with _log_time('stopped %s', self):
# entrypoint have to be stopped before dependencies to ensure
# that running workers can successfully complete
self.entrypoints.all.stop()
# there might still be some running workers, which we have to
# wait for to complete before we can stop dependencies
self._worker_pool.waitall()
# it should be safe now to stop any dependency as there is no
# active worker which could be using it
self.dependencies.all.stop()
# finally, stop remaining extensions
self.subextensions.all.stop()
# any any managed threads they spawned
self._kill_managed_threads()
self.started = False
# if `kill` is called after `stop`, they race to send this
if not self._died.ready():
self._died.send(None)
def kill(self, exc_info=None):
""" Kill the container in a semi-graceful way.
Entrypoints are killed, followed by any active worker threads.
Next, dependencies are killed. Finally, any remaining managed threads
are killed.
If ``exc_info`` is provided, the exception will be raised by
:meth:`~wait``.
"""
if self._being_killed:
# this happens if a managed thread exits with an exception
# while the container is being killed or if multiple errors
# happen simultaneously
_log.debug('already killing %s ... waiting for death', self)
try:
self._died.wait()
except:
pass # don't re-raise if we died with an exception
return
self._being_killed = True
if self._died.ready():
_log.debug('already stopped %s', self)
return
if exc_info is not None:
_log.info('killing %s due to %s', self, exc_info[1])
else:
_log.info('killing %s', self)
# protect against extensions that throw during kill; the container
# is already dying with an exception, so ignore anything else
def safely_kill_extensions(ext_set):
try:
ext_set.kill()
except Exception as exc:
_log.warning('Extension raised `%s` during kill', exc)
safely_kill_extensions(self.entrypoints.all)
self._kill_worker_threads()
safely_kill_extensions(self.extensions.all)
self._kill_managed_threads()
self.started = False
# if `kill` is called after `stop`, they race to send this
if not self._died.ready():
self._died.send(None, exc_info)
def wait(self):
""" Block until the container has been stopped.
If the container was stopped due to an exception, ``wait()`` will
raise it.
Any unhandled exception raised in a managed thread or in the
worker lifecycle (e.g. inside :meth:`DependencyProvider.worker_setup`)
results in the container being ``kill()``ed, and the exception
raised from ``wait()``.
"""
return self._died.wait()
def spawn_worker(self, entrypoint, args, kwargs,
context_data=None, handle_result=None):
""" Spawn a worker thread for running the service method decorated
by `entrypoint`.
``args`` and ``kwargs`` are used as parameters for the service method.
``context_data`` is used to initialize a ``WorkerContext``.
``handle_result`` is an optional function which may be passed
in by the entrypoint. It is called with the result returned
or error raised by the service method. If provided it must return a
value for ``result`` and ``exc_info`` to propagate to dependencies;
these may be different to those returned by the service method.
"""
if self._being_killed:
_log.info("Worker spawn prevented due to being killed")
raise ContainerBeingKilled()
service = self.service_cls()
worker_ctx = WorkerContext(
self, service, entrypoint, args, kwargs, data=context_data
)
_log.debug('spawning %s', worker_ctx)
gt = self._worker_pool.spawn(
self._run_worker, worker_ctx, handle_result
)
gt.link(self._handle_worker_thread_exited, worker_ctx)
self._worker_threads[worker_ctx] = gt
return worker_ctx
def spawn_managed_thread(self, fn, identifier=None):
""" Spawn a managed thread to run ``fn`` on behalf of an extension.
The passed `identifier` will be included in logs related to this
thread, and otherwise defaults to `fn.__name__`, if it is set.
Any uncaught errors inside ``fn`` cause the container to be killed.
It is the caller's responsibility to terminate their spawned threads.
Threads are killed automatically if they are still running after
all extensions are stopped during :meth:`ServiceContainer.stop`.
Extensions should delegate all thread spawning to the container.
"""
if identifier is None:
identifier = getattr(fn, '__name__', "<unknown>")
gt = eventlet.spawn(fn)
self._managed_threads[gt] = identifier
gt.link(self._handle_managed_thread_exited, identifier)
return gt
def _run_worker(self, worker_ctx, handle_result):
_log.debug('setting up %s', worker_ctx)
_log.debug('call stack for %s: %s',
worker_ctx, '->'.join(worker_ctx.call_id_stack))
with _log_time('ran worker %s', worker_ctx):
self._inject_dependencies(worker_ctx)
self._worker_setup(worker_ctx)
result = exc_info = None
method_name = worker_ctx.entrypoint.method_name
method = getattr(worker_ctx.service, method_name)
try:
_log.debug('calling handler for %s', worker_ctx)
with _log_time('ran handler for %s', worker_ctx):
result = method(*worker_ctx.args, **worker_ctx.kwargs)
except Exception as exc:
if isinstance(exc, worker_ctx.entrypoint.expected_exceptions):
_log.warning(
'(expected) error handling worker %s: %s',
worker_ctx, exc, exc_info=True)
else:
_log.exception(
'error handling worker %s: %s', worker_ctx, exc)
exc_info = sys.exc_info()
if handle_result is not None:
_log.debug('handling result for %s', worker_ctx)
with _log_time('handled result for %s', worker_ctx):
result, exc_info = handle_result(
worker_ctx, result, exc_info)
with _log_time('tore down worker %s', worker_ctx):
self._worker_result(worker_ctx, result, exc_info)
# we don't need this any more, and breaking the cycle means
# this can be reclaimed immediately, rather than waiting for a
# gc sweep
del exc_info
self._worker_teardown(worker_ctx)
def _inject_dependencies(self, worker_ctx):
for provider in self.dependencies:
dependency = provider.get_dependency(worker_ctx)
setattr(worker_ctx.service, provider.attr_name, dependency)
def _worker_setup(self, worker_ctx):
for provider in self.dependencies:
provider.worker_setup(worker_ctx)
def _worker_result(self, worker_ctx, result, exc_info):
_log.debug('signalling result for %s', worker_ctx)
for provider in self.dependencies:
provider.worker_result(worker_ctx, result, exc_info)
def _worker_teardown(self, worker_ctx):
for provider in self.dependencies:
provider.worker_teardown(worker_ctx)
def _kill_worker_threads(self):
""" Kill any currently executing worker threads.
See :meth:`ServiceContainer.spawn_worker`
"""
num_workers = len(self._worker_threads)
if num_workers:
_log.warning('killing %s active workers(s)', num_workers)
for worker_ctx, gt in list(self._worker_threads.items()):
_log.warning('killing active worker for %s', worker_ctx)
gt.kill()
def _kill_managed_threads(self):
""" Kill any currently executing managed threads.
See :meth:`ServiceContainer.spawn_managed_thread`
"""
num_threads = len(self._managed_threads)
if num_threads:
_log.warning('killing %s managed thread(s)', num_threads)
for gt, identifier in list(self._managed_threads.items()):
_log.warning('killing managed thread `%s`', identifier)
gt.kill()
def _handle_worker_thread_exited(self, gt, worker_ctx):
self._worker_threads.pop(worker_ctx, None)
self._handle_thread_exited(gt)
def _handle_managed_thread_exited(self, gt, extension):
self._managed_threads.pop(gt, None)
self._handle_thread_exited(gt)
def _handle_thread_exited(self, gt):
try:
gt.wait()
except GreenletExit:
# we don't care much about threads killed by the container
# this can happen in stop() and kill() if extensions
# don't properly take care of their threads
_log.debug('%s thread killed by container', self)
except Exception:
_log.critical('%s thread exited with error', self, exc_info=True)
# any uncaught error in a thread is unexpected behavior
# and probably a bug in the extension or container.
# to be safe we call self.kill() to kill our dependencies and
# provide the exception info to be raised in self.wait().
self.kill(sys.exc_info())
def __repr__(self):
service_name = self.service_name
return '<ServiceContainer [{}] at 0x{:x}>'.format(
service_name, id(self))
def entrance():
pool = GreenPool(100)
for x in range(10):
pool.spawn(fetcher)
pool.waitall()
class Checker(object):
def __init__(self,
namespace,
concurrency=50,
error_file=None,
rebuild_file=None,
full=True):
self.pool = GreenPool(concurrency)
self.error_file = error_file
self.full = bool(full)
if self.error_file:
f = open(self.error_file, 'a')
self.error_writer = csv.writer(f, delimiter=' ')
self.rebuild_file = rebuild_file
if self.rebuild_file:
fd = open(self.rebuild_file, 'a')
self.rebuild_writer = csv.writer(fd, delimiter='|')
conf = {'namespace': namespace}
self.account_client = AccountClient(conf)
self.container_client = ContainerClient(conf)
self.blob_client = BlobClient()
self.accounts_checked = 0
self.containers_checked = 0
self.objects_checked = 0
self.chunks_checked = 0
self.account_not_found = 0
self.container_not_found = 0
self.object_not_found = 0
self.chunk_not_found = 0
self.account_exceptions = 0
self.container_exceptions = 0
self.object_exceptions = 0
self.chunk_exceptions = 0
self.list_cache = {}
self.running = {}
def write_error(self, target):
error = [target.account]
if target.container:
error.append(target.container)
if target.obj:
error.append(target.obj)
if target.chunk:
error.append(target.chunk)
self.error_writer.writerow(error)
def write_rebuilder_input(self, target, obj_meta, ct_meta):
try:
cid = ct_meta['system']['sys.name'].split('.', 1)[0]
except KeyError:
cid = ct_meta['properties']['sys.name'].split('.', 1)[0]
self.rebuild_writer.writerow((cid, obj_meta['id'], target.chunk))
def write_chunk_error(self, target, obj_meta, chunk=None):
if chunk is not None:
target = target.copy()
target.chunk = chunk
if self.error_file:
self.write_error(target)
if self.rebuild_file:
self.write_rebuilder_input(
target, obj_meta,
self.list_cache[(target.account, target.container)][1])
def _check_chunk_xattr(self, target, obj_meta, xattr_meta):
error = False
# Composed position -> erasure coding
attr_prefix = 'meta' if '.' in obj_meta['pos'] else ''
attr_key = attr_prefix + 'chunk_size'
if str(obj_meta['size']) != xattr_meta.get(attr_key):
print(
" Chunk %s '%s' xattr (%s) "
"differs from size in meta2 (%s)" %
(target, attr_key, xattr_meta.get(attr_key), obj_meta['size']))
error = True
attr_key = attr_prefix + 'chunk_hash'
if obj_meta['hash'] != xattr_meta.get(attr_key):
print(
" Chunk %s '%s' xattr (%s) "
"differs from hash in meta2 (%s)" %
(target, attr_key, xattr_meta.get(attr_key), obj_meta['hash']))
error = True
return error
def check_chunk(self, target):
chunk = target.chunk
obj_listing, obj_meta = self.check_obj(target)
error = False
if chunk not in obj_listing:
print(' Chunk %s missing from object listing' % target)
error = True
db_meta = dict()
else:
db_meta = obj_listing[chunk]
try:
xattr_meta = self.blob_client.chunk_head(chunk, xattr=self.full)
except exc.NotFound as e:
self.chunk_not_found += 1
error = True
print(' Not found chunk "%s": %s' % (target, str(e)))
except Exception as e:
self.chunk_exceptions += 1
error = True
print(' Exception chunk "%s": %s' % (target, str(e)))
else:
if db_meta and self.full:
error = self._check_chunk_xattr(target, db_meta, xattr_meta)
if error:
self.write_chunk_error(target, obj_meta)
self.chunks_checked += 1
def check_obj_policy(self, target, obj_meta, chunks):
"""
Check that the list of chunks of an object matches
the object's storage policy.
"""
stg_met = STORAGE_METHODS.load(obj_meta['chunk_method'])
chunks_by_pos = _sort_chunks(chunks, stg_met.ec)
if stg_met.ec:
required = stg_met.ec_nb_data + stg_met.ec_nb_parity
else:
required = stg_met.nb_copy
for pos, clist in chunks_by_pos.iteritems():
if len(clist) < required:
print(' Missing %d chunks at position %s of %s' %
(required - len(clist), pos, target))
if stg_met.ec:
subs = {x['num'] for x in clist}
for sub in range(required):
if sub not in subs:
self.write_chunk_error(target, obj_meta,
'%d.%d' % (pos, sub))
else:
self.write_chunk_error(target, obj_meta, str(pos))
def check_obj(self, target, recurse=False):
account = target.account
container = target.container
obj = target.obj
if (account, container, obj) in self.running:
self.running[(account, container, obj)].wait()
if (account, container, obj) in self.list_cache:
return self.list_cache[(account, container, obj)]
self.running[(account, container, obj)] = Event()
print('Checking object "%s"' % target)
container_listing, ct_meta = self.check_container(target)
error = False
if obj not in container_listing:
print(' Object %s missing from container listing' % target)
error = True
# checksum = None
else:
# TODO check checksum match
# checksum = container_listing[obj]['hash']
pass
results = []
meta = dict()
try:
meta, results = self.container_client.content_locate(
account=account, reference=container, path=obj)
except exc.NotFound as e:
self.object_not_found += 1
error = True
print(' Not found object "%s": %s' % (target, str(e)))
except Exception as e:
self.object_exceptions += 1
error = True
print(' Exception object "%s": %s' % (target, str(e)))
chunk_listing = dict()
for chunk in results:
chunk_listing[chunk['url']] = chunk
self.check_obj_policy(target.copy(), meta, results)
self.objects_checked += 1
self.list_cache[(account, container, obj)] = (chunk_listing, meta)
self.running[(account, container, obj)].send(True)
del self.running[(account, container, obj)]
if recurse:
for chunk in chunk_listing:
t = target.copy()
t.chunk = chunk
self.pool.spawn_n(self.check_chunk, t)
if error and self.error_file:
self.write_error(target)
return chunk_listing, meta
def check_container(self, target, recurse=False):
account = target.account
container = target.container
if (account, container) in self.running:
self.running[(account, container)].wait()
if (account, container) in self.list_cache:
return self.list_cache[(account, container)]
self.running[(account, container)] = Event()
print('Checking container "%s"' % target)
account_listing = self.check_account(target)
error = False
if container not in account_listing:
error = True
print(' Container %s missing from account listing' % target)
marker = None
results = []
ct_meta = dict()
while True:
try:
_, resp = self.container_client.content_list(
account=account, reference=container, marker=marker)
except exc.NotFound as e:
self.container_not_found += 1
error = True
print(' Not found container "%s": %s' % (target, str(e)))
break
except Exception as e:
self.container_exceptions += 1
error = True
print(' Exception container "%s": %s' % (target, str(e)))
break
if resp['objects']:
marker = resp['objects'][-1]['name']
results.extend(resp['objects'])
else:
ct_meta = resp
ct_meta.pop('objects')
break
container_listing = dict()
for obj in results:
container_listing[obj['name']] = obj
self.containers_checked += 1
self.list_cache[(account, container)] = container_listing, ct_meta
self.running[(account, container)].send(True)
del self.running[(account, container)]
if recurse:
for obj in container_listing:
t = target.copy()
t.obj = obj
self.pool.spawn_n(self.check_obj, t, True)
if error and self.error_file:
self.write_error(target)
return container_listing, ct_meta
def check_account(self, target, recurse=False):
account = target.account
if account in self.running:
self.running[account].wait()
if account in self.list_cache:
return self.list_cache[account]
self.running[account] = Event()
print('Checking account "%s"' % target)
error = False
marker = None
results = []
while True:
try:
resp = self.account_client.container_list(account,
marker=marker)
except Exception as e:
self.account_exceptions += 1
error = True
print(' Exception account "%s": %s' % (target, str(e)))
break
if resp['listing']:
marker = resp['listing'][-1][0]
else:
break
results.extend(resp['listing'])
containers = dict()
for e in results:
containers[e[0]] = (e[1], e[2])
self.list_cache[account] = containers
self.running[account].send(True)
del self.running[account]
self.accounts_checked += 1
if recurse:
for container in containers:
t = target.copy()
t.container = container
self.pool.spawn_n(self.check_container, t, True)
if error and self.error_file:
self.write_error(target)
return containers
def check(self, target):
if target.chunk and target.obj and target.container:
self.pool.spawn_n(self.check_chunk, target)
elif target.obj and target.container:
self.pool.spawn_n(self.check_obj, target, True)
elif target.container:
self.pool.spawn_n(self.check_container, target, True)
else:
self.pool.spawn_n(self.check_account, target, True)
def wait(self):
self.pool.waitall()
def report(self):
def _report_stat(name, stat):
print("{0:18}: {1}".format(name, stat))
print()
print('Report')
_report_stat("Accounts checked", self.accounts_checked)
if self.account_not_found:
_report_stat("Missing accounts", self.account_not_found)
if self.account_exceptions:
_report_stat("Exceptions", self.account_not_found)
print()
_report_stat("Containers checked", self.containers_checked)
if self.container_not_found:
_report_stat("Missing containers", self.container_not_found)
if self.container_exceptions:
_report_stat("Exceptions", self.container_exceptions)
print()
_report_stat("Objects checked", self.objects_checked)
if self.object_not_found:
_report_stat("Missing objects", self.object_not_found)
if self.object_exceptions:
_report_stat("Exceptions", self.object_exceptions)
print()
_report_stat("Chunks checked", self.chunks_checked)
if self.chunk_not_found:
_report_stat("Missing chunks", self.chunk_not_found)
if self.chunk_exceptions:
_report_stat("Exceptions", self.chunk_exceptions)
class ServiceContainer(object):
def __init__(self, service_cls, worker_ctx_cls, config):
self.service_cls = service_cls
self.worker_ctx_cls = worker_ctx_cls
self.service_name = get_service_name(service_cls)
self.config = config
self.max_workers = (
config.get(MAX_WORKERS_CONFIG_KEY) or DEFAULT_MAX_WORKERS)
self.dependencies = DependencySet()
for dep in prepare_dependencies(self):
self.dependencies.add(dep)
self.started = False
self._worker_pool = GreenPool(size=self.max_workers)
self._active_threads = {}
self._protected_threads = set()
self._being_killed = False
self._died = Event()
@property
def entrypoints(self):
return filter(is_entrypoint_provider, self.dependencies)
@property
def injections(self):
return filter(is_injection_provider, self.dependencies)
def start(self):
""" Start a container by starting all the dependency providers.
"""
_log.debug('starting %s', self)
self.started = True
with _log_time('started %s', self):
self.dependencies.all.prepare()
self.dependencies.all.start()
def stop(self):
""" Stop the container gracefully.
First all entrypoints are asked to ``stop()``.
This ensures that no new worker threads are started.
It is the providers' responsibility to gracefully shut down when
``stop()`` is called on them and only return when they have stopped.
After all entrypoints have stopped the container waits for any
active workers to complete.
After all active workers have stopped the container stops all
injections.
At this point there should be no more managed threads. In case there
are any managed threads, they are killed by the container.
"""
if self._died.ready():
_log.debug('already stopped %s', self)
return
if self._being_killed:
# this race condition can happen when a container is hosted by a
# runner and yields during its kill method; if it's unlucky in
# scheduling the runner will try to stop() it before self._died
# has a result
_log.debug('already being killed %s', self)
try:
self._died.wait()
except:
pass # don't re-raise if we died with an exception
return
_log.debug('stopping %s', self)
with _log_time('stopped %s', self):
dependencies = self.dependencies
# entrypoint deps have to be stopped before injection deps
# to ensure that running workers can successfully complete
dependencies.entrypoints.all.stop()
# there might still be some running workers, which we have to
# wait for to complete before we can stop injection dependencies
self._worker_pool.waitall()
# it should be safe now to stop any injection as there is no
# active worker which could be using it
dependencies.injections.all.stop()
# finally, stop nested dependencies
dependencies.nested.all.stop()
# just in case there was a provider not taking care of its workers,
# or a dependency not taking care of its protected threads
self._kill_active_threads()
self._kill_protected_threads()
self.started = False
self._died.send(None)
def kill(self, exc_info=None):
""" Kill the container in a semi-graceful way.
All non-protected managed threads are killed first. This includes
all active workers generated by :meth:`ServiceContainer.spawn_worker`.
Next, dependencies are killed. Finally, any remaining protected threads
are killed.
If ``exc_info`` is provided, the exception will be raised by
:meth:`~wait``.
"""
if self._being_killed:
# this happens if a managed thread exits with an exception
# while the container is being killed or if multiple errors
# happen simultaneously
_log.debug('already killing %s ... waiting for death', self)
try:
self._died.wait()
except:
pass # don't re-raise if we died with an exception
return
self._being_killed = True
if self._died.ready():
_log.debug('already stopped %s', self)
return
if exc_info is not None:
_log.info('killing %s due to %s', self, exc_info[1])
else:
_log.info('killing %s', self)
# protect against dependencies that throw during kill; the container
# is already dying with an exception, so ignore anything else
def safely_kill_dependencies(dep_set):
try:
dep_set.kill()
except Exception as exc:
_log.warning('Dependency raised `%s` during kill', exc)
safely_kill_dependencies(self.dependencies.entrypoints.all)
self._kill_active_threads()
safely_kill_dependencies(self.dependencies.all)
self._kill_protected_threads()
self.started = False
self._died.send(None, exc_info)
def wait(self):
""" Block until the container has been stopped.
If the container was stopped due to an exception, ``wait()`` will
raise it.
Any unhandled exception raised in a managed thread or in the
life-cycle management code also causes the container to be
``kill()``ed, which causes an exception to be raised from ``wait()``.
"""
return self._died.wait()
def spawn_worker(self, provider, args, kwargs,
context_data=None, handle_result=None):
""" Spawn a worker thread for running the service method decorated
with an entrypoint ``provider``.
``args`` and ``kwargs`` are used as arguments for the service
method.
``context_data`` is used to initialize a ``WorkerContext``.
``handle_result`` is an optional function which may be passed
in by the entrypoint provider. It is called with the result returned
or error raised by the service method. If provided it must return a
value for ``result`` and ``exc_info`` to propagate to dependencies;
these may be different to those returned by the service method.
"""
if self._being_killed:
_log.info("Worker spawn prevented due to being killed")
raise ContainerBeingKilled()
service = self.service_cls()
worker_ctx = self.worker_ctx_cls(
self, service, provider, args, kwargs, data=context_data)
_log.debug('spawning %s', worker_ctx)
gt = self._worker_pool.spawn(self._run_worker, worker_ctx,
handle_result)
self._active_threads[gt] = provider
gt.link(self._handle_thread_exited)
return worker_ctx
def spawn_managed_thread(self, run_method, protected=False):
""" Spawn a managed thread to run ``run_method``.
Threads can be marked as ``protected``, which means the container will
not forcibly kill them until after all dependencies have been killed.
Dependencies that require a managed thread to complete their kill
procedure should ensure to mark them as ``protected``.
Any uncaught errors inside ``run_method`` cause the container to be
killed.
It is the caller's responsibility to terminate their spawned threads.
Threads are killed automatically if they are still running after
all dependencies are stopped during :meth:`ServiceContainer.stop`.
Entrypoints may only create separate threads using this method,
to ensure they are life-cycle managed.
"""
gt = eventlet.spawn(run_method)
if not protected:
self._active_threads[gt] = MANAGED_THREAD
else:
self._protected_threads.add(gt)
gt.link(self._handle_thread_exited)
return gt
def _run_worker(self, worker_ctx, handle_result):
_log.debug('setting up %s', worker_ctx)
if not worker_ctx.parent_call_stack:
_log.debug('starting call chain')
_log.debug('call stack for %s: %s',
worker_ctx, '->'.join(worker_ctx.call_id_stack))
with _log_time('ran worker %s', worker_ctx):
self.dependencies.injections.all.inject(worker_ctx)
self.dependencies.all.worker_setup(worker_ctx)
result = exc_info = None
method = getattr(worker_ctx.service, worker_ctx.provider.name)
try:
_log.debug('calling handler for %s', worker_ctx)
with _log_time('ran handler for %s', worker_ctx):
result = method(*worker_ctx.args, **worker_ctx.kwargs)
except Exception as exc:
_log.debug('error handling worker %s: %s', worker_ctx, exc,
exc_info=True)
exc_info = sys.exc_info()
if handle_result is not None:
_log.debug('handling result for %s', worker_ctx)
with _log_time('handled result for %s', worker_ctx):
result, exc_info = handle_result(
worker_ctx, result, exc_info)
with _log_time('tore down worker %s', worker_ctx):
_log.debug('signalling result for %s', worker_ctx)
self.dependencies.injections.all.worker_result(
worker_ctx, result, exc_info)
# we don't need this any more, and breaking the cycle means
# this can be reclaimed immediately, rather than waiting for a
# gc sweep
del exc_info
self.dependencies.all.worker_teardown(worker_ctx)
self.dependencies.injections.all.release(worker_ctx)
def _kill_active_threads(self):
""" Kill all managed threads that were not marked as "protected" when
they were spawned.
This set will include all worker threads generated by
:meth:`ServiceContainer.spawn_worker`.
See :meth:`ServiceContainer.spawn_managed_thread`
"""
num_active_threads = len(self._active_threads)
if num_active_threads:
_log.warning('killing %s active thread(s)', num_active_threads)
for gt, provider in list(self._active_threads.items()):
if provider is not MANAGED_THREAD:
description = '{}.{}'.format(
self.service_name, provider.name)
_log.warning('killing active thread for %s', description)
gt.kill()
def _kill_protected_threads(self):
""" Kill any managed threads marked as protected when they were
spawned.
See :meth:`ServiceContainer.spawn_managed_thread`
"""
num_protected_threads = len(self._protected_threads)
if num_protected_threads:
_log.warning('killing %s protected thread(s)',
num_protected_threads)
for gt in list(self._protected_threads):
gt.kill()
def _handle_thread_exited(self, gt):
self._active_threads.pop(gt, None)
self._protected_threads.discard(gt)
try:
gt.wait()
except GreenletExit:
# we don't care much about threads killed by the container
# this can happen in stop() and kill() if providers
# don't properly take care of their threads
_log.warning('%s thread killed by container', self)
except Exception:
_log.error('%s thread exited with error', self, exc_info=True)
# any error raised inside an active thread is unexpected behavior
# and probably a bug in the providers or container.
# to be safe we call self.kill() to kill our dependencies and
# provide the exception info to be raised in self.wait().
self.kill(sys.exc_info())
def __str__(self):
return '<ServiceContainer [{}] at 0x{:x}>'.format(
self.service_name, id(self))
class ServiceContainer(object):
def __init__(self, service_cls, config):
self.service_cls = service_cls
self.config = config
self.service_name = get_service_name(service_cls)
self.shared_extensions = {}
self.max_workers = (config.get(MAX_WORKERS_CONFIG_KEY)
or DEFAULT_MAX_WORKERS)
self.serializer = config.get(SERIALIZER_CONFIG_KEY, DEFAULT_SERIALIZER)
self.accept = [self.serializer]
self.entrypoints = SpawningSet()
self.dependencies = SpawningSet()
self.subextensions = SpawningSet()
for attr_name, dependency in inspect.getmembers(
service_cls, is_dependency):
bound = dependency.bind(self.interface, attr_name)
self.dependencies.add(bound)
self.subextensions.update(iter_extensions(bound))
for method_name, method in inspect.getmembers(service_cls, is_method):
entrypoints = getattr(method, ENTRYPOINT_EXTENSIONS_ATTR, [])
for entrypoint in entrypoints:
bound = entrypoint.bind(self.interface, method_name)
self.entrypoints.add(bound)
self.subextensions.update(iter_extensions(bound))
self.started = False
self._worker_pool = GreenPool(size=self.max_workers)
self._worker_threads = {}
self._managed_threads = {}
self._being_killed = False
self._died = Event()
@property
def extensions(self):
return SpawningSet(self.entrypoints | self.dependencies
| self.subextensions)
@property
def interface(self):
""" An interface to this container for use by extensions.
"""
return self
def start(self):
""" Start a container by starting all of its extensions.
"""
_log.debug('starting %s', self)
self.started = True
with _log_time('started %s', self):
self.extensions.all.setup()
self.extensions.all.start()
def stop(self):
""" Stop the container gracefully.
First all entrypoints are asked to ``stop()``.
This ensures that no new worker threads are started.
It is the extensions' responsibility to gracefully shut down when
``stop()`` is called on them and only return when they have stopped.
After all entrypoints have stopped the container waits for any
active workers to complete.
After all active workers have stopped the container stops all
dependency providers.
At this point there should be no more managed threads. In case there
are any managed threads, they are killed by the container.
"""
if self._died.ready():
_log.debug('already stopped %s', self)
return
if self._being_killed:
# this race condition can happen when a container is hosted by a
# runner and yields during its kill method; if it's unlucky in
# scheduling the runner will try to stop() it before self._died
# has a result
_log.debug('already being killed %s', self)
try:
self._died.wait()
except:
pass # don't re-raise if we died with an exception
return
_log.debug('stopping %s', self)
with _log_time('stopped %s', self):
# entrypoint have to be stopped before dependencies to ensure
# that running workers can successfully complete
self.entrypoints.all.stop()
# there might still be some running workers, which we have to
# wait for to complete before we can stop dependencies
self._worker_pool.waitall()
# it should be safe now to stop any dependency as there is no
# active worker which could be using it
self.dependencies.all.stop()
# finally, stop remaining extensions
self.subextensions.all.stop()
# any any managed threads they spawned
self._kill_managed_threads()
self.started = False
# if `kill` is called after `stop`, they race to send this
if not self._died.ready():
self._died.send(None)
def kill(self, exc_info=None):
""" Kill the container in a semi-graceful way.
Entrypoints are killed, followed by any active worker threads.
Next, dependencies are killed. Finally, any remaining managed threads
are killed.
If ``exc_info`` is provided, the exception will be raised by
:meth:`~wait``.
"""
if self._being_killed:
# this happens if a managed thread exits with an exception
# while the container is being killed or if multiple errors
# happen simultaneously
_log.debug('already killing %s ... waiting for death', self)
try:
self._died.wait()
except:
pass # don't re-raise if we died with an exception
return
self._being_killed = True
if self._died.ready():
_log.debug('already stopped %s', self)
return
if exc_info is not None:
_log.info('killing %s due to %s', self, exc_info[1])
else:
_log.info('killing %s', self)
# protect against extensions that throw during kill; the container
# is already dying with an exception, so ignore anything else
def safely_kill_extensions(ext_set):
try:
ext_set.kill()
except Exception as exc:
_log.warning('Extension raised `%s` during kill', exc)
safely_kill_extensions(self.entrypoints.all)
self._kill_worker_threads()
safely_kill_extensions(self.extensions.all)
self._kill_managed_threads()
self.started = False
# if `kill` is called after `stop`, they race to send this
if not self._died.ready():
self._died.send(None, exc_info)
def wait(self):
""" Block until the container has been stopped.
If the container was stopped due to an exception, ``wait()`` will
raise it.
Any unhandled exception raised in a managed thread or in the
worker lifecycle (e.g. inside :meth:`DependencyProvider.worker_setup`)
results in the container being ``kill()``ed, and the exception
raised from ``wait()``.
"""
return self._died.wait()
def spawn_worker(self,
entrypoint,
args,
kwargs,
context_data=None,
handle_result=None):
""" Spawn a worker thread for running the service method decorated
by `entrypoint`.
``args`` and ``kwargs`` are used as parameters for the service method.
``context_data`` is used to initialize a ``WorkerContext``.
``handle_result`` is an optional function which may be passed
in by the entrypoint. It is called with the result returned
or error raised by the service method. If provided it must return a
value for ``result`` and ``exc_info`` to propagate to dependencies;
these may be different to those returned by the service method.
"""
if self._being_killed:
_log.info("Worker spawn prevented due to being killed")
raise ContainerBeingKilled()
service = self.service_cls()
worker_ctx = WorkerContext(self,
service,
entrypoint,
args,
kwargs,
data=context_data)
_log.debug('spawning %s', worker_ctx)
gt = self._worker_pool.spawn(self._run_worker, worker_ctx,
handle_result)
gt.link(self._handle_worker_thread_exited, worker_ctx)
self._worker_threads[worker_ctx] = gt
return worker_ctx
def spawn_managed_thread(self, fn, identifier=None):
""" Spawn a managed thread to run ``fn`` on behalf of an extension.
The passed `identifier` will be included in logs related to this
thread, and otherwise defaults to `fn.__name__`, if it is set.
Any uncaught errors inside ``fn`` cause the container to be killed.
It is the caller's responsibility to terminate their spawned threads.
Threads are killed automatically if they are still running after
all extensions are stopped during :meth:`ServiceContainer.stop`.
Extensions should delegate all thread spawning to the container.
"""
if identifier is None:
identifier = getattr(fn, '__name__', "<unknown>")
gt = eventlet.spawn(fn)
self._managed_threads[gt] = identifier
gt.link(self._handle_managed_thread_exited, identifier)
return gt
def _run_worker(self, worker_ctx, handle_result):
_log.debug('setting up %s', worker_ctx)
_log.debug('call stack for %s: %s', worker_ctx,
'->'.join(worker_ctx.call_id_stack))
with _log_time('ran worker %s', worker_ctx):
self._inject_dependencies(worker_ctx)
self._worker_setup(worker_ctx)
result = exc_info = None
method_name = worker_ctx.entrypoint.method_name
method = getattr(worker_ctx.service, method_name)
try:
_log.debug('calling handler for %s', worker_ctx)
with _log_time('ran handler for %s', worker_ctx):
result = method(*worker_ctx.args, **worker_ctx.kwargs)
except Exception as exc:
_log.info('error handling worker %s: %s',
worker_ctx,
exc,
exc_info=True)
exc_info = sys.exc_info()
if handle_result is not None:
_log.debug('handling result for %s', worker_ctx)
with _log_time('handled result for %s', worker_ctx):
result, exc_info = handle_result(worker_ctx, result,
exc_info)
with _log_time('tore down worker %s', worker_ctx):
self._worker_result(worker_ctx, result, exc_info)
# we don't need this any more, and breaking the cycle means
# this can be reclaimed immediately, rather than waiting for a
# gc sweep
del exc_info
self._worker_teardown(worker_ctx)
def _inject_dependencies(self, worker_ctx):
for provider in self.dependencies:
dependency = provider.get_dependency(worker_ctx)
setattr(worker_ctx.service, provider.attr_name, dependency)
def _worker_setup(self, worker_ctx):
for provider in self.dependencies:
provider.worker_setup(worker_ctx)
def _worker_result(self, worker_ctx, result, exc_info):
_log.debug('signalling result for %s', worker_ctx)
for provider in self.dependencies:
provider.worker_result(worker_ctx, result, exc_info)
def _worker_teardown(self, worker_ctx):
for provider in self.dependencies:
provider.worker_teardown(worker_ctx)
def _kill_worker_threads(self):
""" Kill any currently executing worker threads.
See :meth:`ServiceContainer.spawn_worker`
"""
num_workers = len(self._worker_threads)
if num_workers:
_log.warning('killing %s active workers(s)', num_workers)
for worker_ctx, gt in list(self._worker_threads.items()):
_log.warning('killing active worker for %s', worker_ctx)
gt.kill()
def _kill_managed_threads(self):
""" Kill any currently executing managed threads.
See :meth:`ServiceContainer.spawn_managed_thread`
"""
num_threads = len(self._managed_threads)
if num_threads:
_log.warning('killing %s managed thread(s)', num_threads)
for gt, identifier in list(self._managed_threads.items()):
_log.warning('killing managed thread `%s`', identifier)
gt.kill()
def _handle_worker_thread_exited(self, gt, worker_ctx):
self._worker_threads.pop(worker_ctx, None)
self._handle_thread_exited(gt)
def _handle_managed_thread_exited(self, gt, extension):
self._managed_threads.pop(gt, None)
self._handle_thread_exited(gt)
def _handle_thread_exited(self, gt):
try:
gt.wait()
except GreenletExit:
# we don't care much about threads killed by the container
# this can happen in stop() and kill() if extensions
# don't properly take care of their threads
_log.debug('%s thread killed by container', self)
except Exception:
_log.error('%s thread exited with error', self, exc_info=True)
# any uncaught error in a thread is unexpected behavior
# and probably a bug in the extension or container.
# to be safe we call self.kill() to kill our dependencies and
# provide the exception info to be raised in self.wait().
self.kill(sys.exc_info())
def __repr__(self):
service_name = self.service_name
return '<ServiceContainer [{}] at 0x{:x}>'.format(
service_name, id(self))
import httplib2 directly, httplib2 doesn't natively support cooperative
yielding.
monkey_patch httplib2 will make httplib2 green.
'''
from eventlet.greenpool import GreenPool
import eventlet
import random
import httplib2
# uncomment this line to green httplib2
# httplib2 = eventlet.import_patched('httplib2')
def worker(url):
print "worker " + str(random.random())
h = httplib2.Http()
resp, content = h.request(url, "GET")
return resp
pool = GreenPool(size=10)
results = pool.imap(worker, open("urls.txt", 'r'))
for result in results:
print result
print "done...."
pool.waitall()
def run_once(self, *args, **kwargs):
"""
Executes a single pass, looking for objects to expire.
:param args: Extra args to fulfill the Daemon interface; this daemon
has no additional args.
:param kwargs: Extra keyword args to fulfill the Daemon interface; this
daemon accepts processes and process keyword args.
These will override the values from the config file if
provided.
"""
# This if-clause will be removed when general task queue feature is
# implemented.
if not self.dequeue_from_legacy:
self.logger.info('This node is not configured to dequeue tasks '
'from the legacy queue. This node will '
'not process any expiration tasks. At least '
'one node in your cluster must be configured '
'with dequeue_from_legacy == true.')
return
self.get_process_values(kwargs)
pool = GreenPool(self.concurrency)
self.report_first_time = self.report_last_time = time()
self.report_objects = 0
try:
self.logger.debug('Run begin')
task_account_container_list_to_delete = list()
for task_account, my_index, divisor in \
self.iter_task_accounts_to_expire():
container_count, obj_count = \
self.swift.get_account_info(task_account)
# the task account is skipped if there are no task container
if not container_count:
continue
self.logger.info(
_('Pass beginning for task account %(account)s; '
'%(container_count)s possible containers; '
'%(obj_count)s possible objects') % {
'account': task_account,
'container_count': container_count,
'obj_count': obj_count
})
task_account_container_list = \
[(task_account, task_container) for task_container in
self.iter_task_containers_to_expire(task_account)]
task_account_container_list_to_delete.extend(
task_account_container_list)
# delete_task_iter is a generator to yield a dict of
# task_account, task_container, task_object, delete_timestamp,
# target_path to handle delete actual object and pop the task
# from the queue.
delete_task_iter = \
self.round_robin_order(self.iter_task_to_expire(
task_account_container_list, my_index, divisor))
for delete_task in delete_task_iter:
pool.spawn_n(self.delete_object, **delete_task)
pool.waitall()
for task_account, task_container in \
task_account_container_list_to_delete:
try:
self.swift.delete_container(
task_account,
task_container,
acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
except (Exception, Timeout) as err:
self.logger.exception(
_('Exception while deleting container %(account)s '
'%(container)s %(err)s') % {
'account': task_account,
'container': task_container,
'err': str(err)
})
self.logger.debug('Run end')
self.report(final=True)
except (Exception, Timeout):
self.logger.exception(_('Unhandled exception'))
class Service(ConsumerMixin):
def __init__(self,
controllercls,
connection,
exchange,
topic,
pool=None,
poolsize=1000):
self.nodeid = UIDGEN()
if pool is None:
self.procpool = GreenPool(size=poolsize)
else:
self.procpool = pool
self.connection = connection
self.controller = controllercls()
self.topic = topic
self.greenlet = None
self.messagesem = Semaphore()
self.consume_ready = Event()
node_topic = "{}.{}".format(self.topic, self.nodeid)
self.queues = [
entities.get_topic_queue(exchange, topic),
entities.get_topic_queue(exchange, node_topic),
entities.get_fanout_queue(topic),
]
self._channel = None
self._consumers = None
def start(self):
# self.connection = newrpc.create_connection()
if self.greenlet is not None and not self.greenlet.dead:
raise RuntimeError()
self.greenlet = eventlet.spawn(self.run)
def get_consumers(self, Consumer, channel):
return [
Consumer(self.queues, callbacks=[
self.on_message,
]),
]
def on_consume_ready(self, connection, channel, consumers, **kwargs):
self._consumers = consumers
self._channel = channel
self.consume_ready.send(None)
def on_consume_end(self, connection, channel):
self.consume_ready.reset()
def on_message(self, body, message):
# need a semaphore to stop killing between message ack()
# and spawning process.
with self.messagesem:
self.procpool.spawn(self.handle_request, body)
message.ack()
def handle_request(self, body):
newrpc.process_message(self.connection, self.controller, body)
def wait(self):
try:
self.greenlet.wait()
except greenlet.GreenletExit:
pass
return self.procpool.waitall()
def kill(self):
if self.greenlet is not None and not self.greenlet.dead:
self.should_stop = True
#with self.messagesem:
#self.greenlet.kill()
self.greenlet.wait()
if self._consumers:
for c in self._consumers:
c.cancel()
if self._channel is not None:
self._channel.close()
def link(self, *args, **kwargs):
return self.greenlet.link(*args, **kwargs)
def kill_processes(self):
for g in self.procpool.coroutines_running:
g.kill()