def run(self):
self.start()
while True:
try:
sig = self.SIG_QUEUE.pop(0) if len(self.SIG_QUEUE) else None
if sig is None:
self.scan()
eventlet.spawn_n(self.update)
eventlet.sleep(0.1)
self.sleep()
continue
if sig not in self.SIG_NAMES:
self.log.info("Ignoring unknown signal: %s" % sig)
continue
signame = self.SIG_NAMES.get(sig)
handler = getattr(self, "handle_%s" % signame, None)
if not handler:
log.error("Unhandled signal: %s" % signame)
continue
log.info("handling signal: %s" % signame)
handler()
self.wakeup()
eventlet.sleep(0.1)
except StopIteration:
self.halt()
except KeyboardInterrupt:
self.halt()
except Exception, e:
log.info("unhandled exception in main loop:\n%s" %
traceback.format_exc())
sys.exit(-1)
def _multi_send(method, context, topic, msg, timeout=None,
envelope=False, _msg_id=None):
"""
Wraps the sending of messages,
dispatches to the matchmaker and sends
message to all relevant hosts.
"""
conf = CONF
LOG.debug(_("%(msg)s") % {'msg': ' '.join(map(pformat, (topic, msg)))})
queues = _get_matchmaker().queues(topic)
LOG.debug(_("Sending message(s) to: %s"), queues)
# Don't stack if we have no matchmaker results
if len(queues) == 0:
LOG.warn(_("No matchmaker results. Not casting."))
# While not strictly a timeout, callers know how to handle
# this exception and a timeout isn't too big a lie.
raise rpc_common.Timeout(_("No match from matchmaker."))
# This supports brokerless fanout (addresses > 1)
for queue in queues:
(_topic, ip_addr) = queue
_addr = "tcp://%s:%s" % (ip_addr, conf.rpc_zmq_port)
if method.__name__ == '_cast':
eventlet.spawn_n(method, _addr, context,
_topic, msg, timeout, envelope,
_msg_id)
return
return method(_addr, context, _topic, msg, timeout,
envelope)
def _event(self):
"""Tries to commit event.
If there is an event in Climate DB to be done, do it and change its
status to 'DONE'.
"""
LOG.debug('Trying to get event from DB.')
event = db_api.event_get_first_sorted_by_filters(
sort_key='time',
sort_dir='asc',
filters={'status': 'UNDONE'}
)
if not event:
return
if event['time'] < datetime.datetime.now():
db_api.event_update(event['id'], {'status': 'IN_PROGRESS'})
event_type = event['event_type']
event_fn = getattr(self, event_type, None)
if event_fn is None:
raise exceptions.EventError(error='Event type %s is not '
'supported' % event_type)
try:
eventlet.spawn_n(service_utils.with_empty_context(event_fn),
event['lease_id'], event['id'])
lease = db_api.lease_get(event['lease_id'])
with trusts.create_ctx_from_trust(lease['trust_id']) as ctx:
self._send_notification(lease,
ctx,
events=['event.%s' % event_type])
except Exception:
db_api.event_update(event['id'], {'status': 'ERROR'})
LOG.exception(_('Error occurred while event handling.'))
def _make_app_iter(self, node, source):
"""
Returns an iterator over the contents of the source (via its read
func). There is also quite a bit of cleanup to ensure garbage
collection works and the underlying socket of the source is closed.
:param source: The httplib.Response object this iterator should read
from.
:param node: The node the source is reading from, for logging purposes.
"""
try:
# Spawn reader to read from the source and place in the queue.
# We then drop any reference to the source or node, for garbage
# collection purposes.
queue = Queue(1)
spawn_n(self._make_app_iter_reader, node, source, queue,
self.app.logger.thread_locals)
source = node = None
while True:
chunk = queue.get(timeout=self.app.node_timeout)
if isinstance(chunk, bool): # terminator
success = chunk
if not success:
raise Exception(_('Failed to read all data'
' from the source'))
break
yield chunk
except Empty:
raise ChunkReadTimeout()
except (GeneratorExit, Timeout):
self.app.logger.warn(_('Client disconnected on read'))
except Exception:
self.app.logger.exception(_('Trying to send to client'))
raise
def run(config, section_name):
import eventlet
# Apply monkey_patch at the earliest
eventlet.monkey_patch(thread=False)
'''
Sets up logging for the entire process and then starts the corresponding handler.
- config : Configuration of the application.
- section_name : Name of the section in the config.
'''
logUtils.setup_logging(config)
logger = logUtils.get_logger(__name__)
solr_url = config['DEFAULT']['solr_url']
db_info = session.DBInfo(config['DEFAULT']['nova_db_server'], config['DEFAULT']['nova_db_port'], config['DEFAULT']['nova_db_creds'])
amqp_info = amqp_client.AmqpInfo(config['DEFAULT']['amqp_server'], config['DEFAULT']['amqp_port'], config['DEFAULT']['amqp_creds'])
# pull out name of the handler for the specific core
section = config[section_name]
handlerName = section["handlerName"]
interval = int(section["interval"])
try :
# instantiate the handler
handlerKls = get_class(handlerName)
handler = handlerKls(solr_url, db_info, amqp_info, interval)
logger.info(str(handler) + ' instantiated to perform work.')
# handler is expected to have an on_start & setup_amqp() method to do the real work.
# run on_start on a green thread.
eventlet.spawn_n(handler.on_start)
# call setup_amqp on the main thread so that it keeps the main thread busy.
handler.setup_amqp()
return handler
except Exception as inst:
logger.exception(inst)
def _on_headers(self, data):
eol = data.find("\r\n")
start_line = data[:eol]
method, uri, version = start_line.split(" ")
if not version.startswith("HTTP/"):
raise Exception("Malformed HTTP version in HTTP Request-Line")
headers = HTTPHeaders.parse(data[eol:])
# do we need to proxy this request somewhere else?
host = headers['Host']
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self.address[0])
content_length = headers.get("Content-Length")
if content_length:
content_length = int(content_length)
if content_length > self.stream.max_buffer_size:
raise Exception("Content-Length too long")
if headers.get("Expect") == "100-continue":
self.stream.write("HTTP/1.1 100 (Continue)\r\n\r\n")
eventlet.spawn_n(self.stream.read_bytes, content_length, self._on_request_body)
return
self.request_callback(self._request)
def __init__(self, bitHopper):
"""
Sets up coin difficulties and reads in old difficulties
from file.
"""
#Add Coins
self.diff = {}
for attr_coin in bitHopper.altercoins.itervalues():
self.diff[attr_coin['short_name']] = attr_coin['recent_difficulty']
#Store bitHopper for logging
self.bitHopper = bitHopper
#Read in old diffs
cfg = ConfigParser.ConfigParser()
cfg.read(["diffwebs.cfg"])
#Add diff_sites
self.diff_sites = []
for site in cfg.sections():
self.diff_sites.append(dict(cfg.items(site)))
self.lock = threading.RLock()
eventlet.spawn_n(self.update_difficulty)
def run(self):
eventlet.spawn_n(self.stats_flush)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
addr = (self.listen_addr, self.listen_port)
sock.bind(addr)
buf = 8192
self.logger.info("Listening on %s:%d" % addr)
if self.debug:
print "Listening on %s:%d" % addr
if self.combined_events:
if self.debug:
print "combined_events mode enabled"
while 1:
data, addr = sock.recvfrom(buf)
if not data:
break
else:
for metric in data.split("#"):
self.decode_recvd(metric)
else:
while 1:
data, addr = sock.recvfrom(buf)
if not data:
break
else:
self.decode_recvd(data)
def _poll_sensors_for_results(self, sensor_ids):
"""
Main loop which polls sensor for results and detects dead sensors.
"""
for sensor_id in sensor_ids:
now = int(time.time())
process = self._processes[sensor_id]
status = process.poll()
if status is not None:
# Dead process detected
LOG.info('Process for sensor %s has exited with code %s', sensor_id, status)
sensor = self._sensors[sensor_id]
self._delete_sensor(sensor_id)
self._dispatch_trigger_for_sensor_exit(sensor=sensor,
exit_code=status)
# Try to respawn a dead process (maybe it was a simple failure which can be
# resolved with a restart)
eventlet.spawn_n(self._respawn_sensor, sensor_id=sensor_id, sensor=sensor,
exit_code=status)
else:
sensor_start_time = self._sensor_start_times[sensor_id]
sensor_respawn_count = self._sensor_respawn_counts[sensor_id]
successfuly_started = (now - sensor_start_time) >= SENSOR_SUCCESSFUL_START_THRESHOLD
if successfuly_started and sensor_respawn_count >= 1:
# Sensor has been successfully running more than threshold seconds, clear the
# respawn counter so we can try to restart the sensor if it dies later on
self._sensor_respawn_counts[sensor_id] = 0
def test_closure(self):
def spam_to_me(address):
sock = eventlet.connect(address)
while True:
try:
sock.sendall(b'hello world')
except socket.error as e:
if get_errno(e) == errno.EPIPE:
return
raise
server = eventlet.listen(('127.0.0.1', 0))
sender = eventlet.spawn(spam_to_me, server.getsockname())
client, address = server.accept()
server.close()
def reader():
try:
while True:
data = client.recv(1024)
assert data
except socket.error as e:
# we get an EBADF because client is closed in the same process
# (but a different greenthread)
if get_errno(e) != errno.EBADF:
raise
def closer():
client.close()
reader = eventlet.spawn(reader)
eventlet.spawn_n(closer)
reader.wait()
sender.wait()
def _write_child_when_able(self):
"""
Select on the child's input file descriptor becoming writable, and then write commands to it.
If not successful in writing all the commands, spawn a new greenlet to retry.
"""
LOG.debug("write_child_when_able")
buffer_contents = self._read_from_write_buffer()
if not buffer_contents:
return
try:
r, w, x = select.select([],[self._fd],[])
except Exception, e:
# The next 9 lines are taken from Facebook's Tornado project, which is open-sourced under
# the Apache license.
# Depending on python version and poll implementation,
# different exception types may be thrown and there are
# two ways EINTR might be signaled:
# * e.errno == errno.EINTR
# * e.args is like (errno.EINTR, 'Interrupted system call')
if (getattr(e, 'errno') == errno.EINTR or
(isinstance(getattr(e, 'args'), tuple) and
len(e.args) == 2 and e.args[0] == errno.EINTR)):
LOG.warning("Interrupted system call", exc_info=1)
eventlet.spawn_n(self._write_child_when_able)
else:
LOG.error("Unexpected error on select")
self.mark_for_cleanup()
return
def send_ip_addr_adv_notif(
ns_name, iface_name, address, count=3, log_exception=True):
"""Send advance notification of an IP address assignment.
If the address is in the IPv4 family, send gratuitous ARP.
If the address is in the IPv6 family, no advance notification is
necessary, since the Neighbor Discovery Protocol (NDP), Duplicate
Address Discovery (DAD), and (for stateless addresses) router
advertisements (RAs) are sufficient for address resolution and
duplicate address detection.
:param ns_name: Namespace name which GARPs are gonna be sent from.
:param iface_name: Name of interface which GARPs are gonna be sent from.
:param address: Advertised IP address.
:param count: (Optional) How many GARPs are gonna be sent. Default is 3.
:param log_exception: (Optional) True if possible failures should be logged
on exception level. Otherwise they are logged on
WARNING level. Default is True.
"""
def arping():
_arping(ns_name, iface_name, address, count, log_exception)
if count > 0 and netaddr.IPAddress(address).version == 4:
eventlet.spawn_n(arping)
def serve_forever(self):
while True:
new_connection, address = self.server.accept()
new_writer = new_connection.makefile('w')
eventlet.spawn_n(self.handle_chat,
new_writer,
new_connection.makefile('r'))
def initialize_if_enabled():
backdoor_locals = {
'exit': _dont_use_this, # So we don't exit the entire process
'quit': _dont_use_this, # So we don't exit the entire process
'fo': _find_objects,
'pgt': _print_greenthreads,
'pnt': _print_nativethreads,
}
if CONF.backdoor_port is None:
return None
start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
# NOTE(johannes): The standard sys.displayhook will print the value of
# the last expression and set it to __builtin__._, which overwrites
# the __builtin__._ that gettext sets. Let's switch to using pprint
# since it won't interact poorly with gettext, and it's easier to
# read the output too.
def displayhook(val):
if val is not None:
pprint.pprint(val)
sys.displayhook = displayhook
sock = _listen('localhost', start_port, end_port, eventlet.listen)
# In the case of backdoor port being zero, a port number is assigned by
# listen(). In any case, pull the port number out here.
port = sock.getsockname()[1]
LOG.info(_('Eventlet backdoor listening on %(port)s for process %(pid)d') %
{'port': port, 'pid': os.getpid()})
eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
locals=backdoor_locals)
return port
def queue_event(self, event):
"""Called to queue sending an event with the next batch of events.
Sending events individually, as they occur, has been problematic as it
can result in a flood of sends. Previously, there was a loopingcall
thread that would send batched events on a periodic interval. However,
maintaining a persistent thread in the loopingcall was also
problematic.
This replaces the loopingcall with a mechanism that creates a
short-lived thread on demand whenever an event is queued. That thread
will wait for a lock, send all queued events and then sleep for
'batch_interval' seconds to allow other events to queue up.
This effectively acts as a rate limiter to only allow 1 batch per
'batch_interval' seconds.
:param event: the event that occurred.
"""
if not event:
return
self.pending_events.append(event)
@utils.synchronized(self._lock_identifier)
def synced_send():
self._notify()
# sleeping after send while holding the lock allows subsequent
# events to batch up
eventlet.sleep(self.batch_interval)
eventlet.spawn_n(synced_send)
def __init__(self, bitHopper):
self.bitHopper = bitHopper
self.lock = threading.RLock()
eventlet.spawn_n(self.update_difficulty)
self.rate = {'btc':1.0}
self.profitability = {'btc':1.0}
def _child_process(self, service):
# Setup child signal handlers differently
def _sigterm(*args):
signal.signal(signal.SIGTERM, signal.SIG_DFL)
raise SignalExit(signal.SIGTERM)
signal.signal(signal.SIGTERM, _sigterm)
# Block SIGINT and let the parent send us a SIGTERM
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Reopen the eventlet hub to make sure we don't share an epoll
# fd with parent and/or siblings, which would be bad
eventlet.hubs.use_hub()
# Close write to ensure only parent has it open
os.close(self.writepipe)
# Create greenthread to watch for parent to close pipe
eventlet.spawn_n(self._pipe_watcher)
# Reseed random number generator
random.seed()
launcher = Launcher()
launcher.launch_service(service)
launcher.wait()
def initialize_if_enabled():
backdoor_locals = {
'exit': _dont_use_this, # So we don't exit the entire process
'quit': _dont_use_this, # So we don't exit the entire process
'fo': _find_objects,
'pgt': _print_greenthreads,
}
if CONF.backdoor_port is None:
return None
# NOTE(johannes): The standard sys.displayhook will print the value of
# the last expression and set it to __builtin__._, which overwrites
# the __builtin__._ that gettext sets. Let's switch to using pprint
# since it won't interact poorly with gettext, and it's easier to
# read the output too.
def displayhook(val):
if val is not None:
pprint.pprint(val)
sys.displayhook = displayhook
sock = eventlet.listen(('localhost', CONF.backdoor_port))
port = sock.getsockname()[1]
eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
locals=backdoor_locals)
return port
def after_start(self):
eventlet.spawn_n(self._process_routers_loop)
LOG.info(_LI("L3 agent started"))
# Do the report state before we do the first full sync.
self._report_state()
self.pd.after_start()
def spawn_n(func, *args, **kwargs):
"""Passthrough method for eventlet.spawn_n.
This utility exists so that it can be stubbed for testing without
interfering with the service spawns.
"""
eventlet.spawn_n(func, *args, **kwargs)
def communicate(self):
self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke')
self.storlet_logger.open()
self._prepare_invocation_descriptors()
try:
self._invoke()
except Exception as e:
raise e
finally:
self._close_remote_side_descriptors()
self.storlet_logger.close()
self._wait_for_write_with_timeout(self.input_data_write_fd)
# We do the writing in a different thread.
# Otherwise, we can run into the following deadlock
# 1. md writeds to Storlet
# 2. Storlet reads and starts to write md and thed data
# 3. md continues writing
# 4. Storlet continues writing and gets stuck as md is busy writing,
# not consuming the reader end of the Storlet writer.
eventlet.spawn_n(self._write_input_data)
out_md = self._read_metadata()
self._wait_for_read_with_timeout(self.data_read_fd)
return out_md, self.data_read_fd
def hook(*args, **kwargs):
hook_result = event.Event()
def wait_for_entrypoint():
with entrypoint_waiter(container, method_name) as waiter_result:
container.spawn_worker(
entrypoint, args, kwargs,
context_data=context_data
)
try:
hook_result.send(waiter_result.get())
except Exception as exc:
hook_result.send_exception(exc)
def wait_for_container():
try:
container.wait()
except Exception as exc:
if not hook_result.ready():
hook_result.send_exception(exc)
# If the container errors (e.g. due to a bad entrypoint), the
# entrypoint_waiter never completes. To mitigate, we also wait on
# the container, and if that throws we send the exception back
# as our result.
eventlet.spawn_n(wait_for_entrypoint)
eventlet.spawn_n(wait_for_container)
return hook_result.wait()
def test_reset(self):
evt = eventlet.Event()
# calling reset before send should throw
self.assertRaises(AssertionError, evt.reset)
value = 'some stuff'
def send_to_event():
evt.send(value)
eventlet.spawn_n(send_to_event)
self.assertEqual(evt.wait(), value)
# now try it again, and we should get the same exact value,
# and we shouldn't be allowed to resend without resetting
value2 = 'second stuff'
self.assertRaises(AssertionError, evt.send, value2)
self.assertEqual(evt.wait(), value)
# reset and everything should be happy
evt.reset()
def send_to_event2():
evt.send(value2)
eventlet.spawn_n(send_to_event2)
self.assertEqual(evt.wait(), value2)
def start(self):
"""Run each worker in new thread"""
for i in range(self.conf.worker_threads):
eventlet.spawn_n(Worker().run)
while True:
self.server.listen()
def test_connect_ssl(self):
def accept_once(listenfd):
try:
conn, addr = listenfd.accept()
conn.write(b'hello\r\n')
greenio.shutdown_safe(conn)
conn.close()
finally:
greenio.shutdown_safe(listenfd)
listenfd.close()
server = eventlet.wrap_ssl(
eventlet.listen(('0.0.0.0', 0)),
self.private_key_file,
self.certificate_file,
server_side=True
)
eventlet.spawn_n(accept_once, server)
raw_client = eventlet.connect(('127.0.0.1', server.getsockname()[1]))
client = ssl.wrap_socket(raw_client)
fd = client.makefile('rb', 8192)
assert fd.readline() == b'hello\r\n'
try:
self.assertEqual(b'', fd.read(10))
except greenio.SSL.ZeroReturnError:
# if it's a GreenSSL object it'll do this
pass
greenio.shutdown_safe(client)
client.close()
check_hub()
def __init__(self, bitHopper):
self.bitHopper = bitHopper
self.interval = 600
self.parseConfig()
self.log_msg(" - Payouts interval: " + str(self.interval))
eventlet.spawn_n(self.run)
self.lock = threading.RLock()
def run_server(self):
"""Run a WSGI server."""
eventlet.wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
eventlet.hubs.use_hub('poll')
eventlet.patcher.monkey_patch(all=False, socket=True)
self.pool = eventlet.GreenPool(size=self.threads)
socket_timeout = cfg.CONF.eventlet_opts.client_socket_timeout or None
# Close write to ensure only parent has it open
os.close(self.writepipe)
# Create greenthread to watch for parent to close pipe
eventlet.spawn_n(self._pipe_watcher)
try:
eventlet.wsgi.server(
self.sock,
self.application,
custom_pool=self.pool,
url_length_limit=URL_LENGTH_LIMIT,
log=self._logger,
debug=cfg.CONF.debug,
keepalive=cfg.CONF.eventlet_opts.wsgi_keep_alive,
socket_timeout=socket_timeout)
except socket.error as err:
if err[0] != errno.EINVAL:
raise
self.pool.waitall()
def notify(action, alarm_id, previous, current, reason, reason_data):
LOG.info(_(
"Notifying alarm %(alarm_id)s from %(previous)s "
"to %(current)s with action %(action)s because "
"%(reason)s") % ({'alarm_id': alarm_id, 'previous': previous,
'current': current, 'action': action,
'reason': reason}))
body = {'alarm_id': alarm_id, 'previous': previous,
'current': current, 'reason': reason,
'reason_data': reason_data}
kwargs = {'data': jsonutils.dumps(body)}
if action.scheme == 'https':
default_verify = int(cfg.CONF.alarm.rest_notifier_ssl_verify)
options = urlparse.parse_qs(action.query)
verify = bool(int(options.get('ceilometer-alarm-ssl-verify',
[default_verify])[-1]))
kwargs['verify'] = verify
cert = cfg.CONF.alarm.rest_notifier_certificate_file
key = cfg.CONF.alarm.rest_notifier_certificate_key
if cert:
kwargs['cert'] = (cert, key) if key else cert
eventlet.spawn_n(requests.post, action.geturl(), **kwargs)
def get_listener():
global _listener
if not _listener:
with Connection(cfg.CONF.messaging.url) as conn:
_listener = Listener(conn)
eventlet.spawn_n(listen, _listener)
return _listener
def _treat_devices_added(self):
try:
devices_details_list = self.plugin_rpc.get_devices_details_list(
self.context,
self._added_ports,
self.agent_id)
except Exception as e:
LOG.debug("Unable to get ports details for "
"devices %(devices)s: %(e)s",
{'devices': self._added_ports, 'e': e})
return
for device_details in devices_details_list:
device = device_details['device']
LOG.info(_LI("Adding port %s"), device)
if 'port_id' in device_details:
LOG.info(_LI("Port %(device)s updated. Details: "
"%(device_details)s"),
{'device': device, 'device_details': device_details})
eventlet.spawn_n(self._process_added_port, device_details)
# remove the port from added ports set, so it doesn't get
# reprocessed.
self._added_ports.discard(device)
def __connectService(self):
try:
self.log('Connecting to minitouch service')
self.log('pid5:%s' % self.pid)
self.minitouchService.forward_minitouch()
self.log('pid3:%s' % self.pid)
self.log('addr:%s' % self.addr[1])
self.createSocket()
self.log('pid4:%s' % self.pid)
if self.get_status == 0 and self.send_status == 0:
self.send_status = 1
t2 = eventlet.spawn_n(self.send)
self.get_status = 1
# time.sleep(2)
t1 = eventlet.spawn_n(self.getInfo)
else:
raise TouchError(
'get:%s/send:%s Error' %
(self.get_status, self.send_status), '__connectService')
except Exception as e:
raise TouchError(str(e), '__connectService')
def _tlshandler(bind_host, bind_port):
global plainsocket
plainsocket = socket.socket(socket.AF_INET6)
plainsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
plainsocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
bound = False
while not bound:
try:
plainsocket.bind((bind_host, bind_port, 0, 0))
bound = True
except socket.error as e:
if e.errno != 98:
raise
sys.stderr.write('TLS Socket in use, retrying in 1 second\n')
eventlet.sleep(1)
# Enable TCP_FASTOPEN
plainsocket.setsockopt(socket.SOL_TCP, 23, 5)
plainsocket.listen(5)
while (1): # TODO: exithook
cnn, addr = plainsocket.accept()
eventlet.spawn_n(_tlsstartup, cnn)
def play(self, block=False):
if self.paused:
_pandora.resume()
self.paused = False
return
def load_and_play():
self.length = _pandora.play(self._download())
self.publish_message("playing %s" % self)
while True:
stats = _pandora.stats()
if stats:
total, pos = stats
if pos == total and account.current_station: break
time.sleep(1)
self.publish_message("finished playing %s" % self)
if block: load_and_play()
else: eventlet.spawn_n(load_and_play)
def test_backdoor_port_range_one_inuse(self):
self.config(backdoor_port='8800:8900')
sock = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(eventlet, 'listen')
self.mox.StubOutWithMock(eventlet, 'spawn_n')
eventlet.listen(
('localhost', 8800)).AndRaise(socket.error(errno.EADDRINUSE, ''))
eventlet.listen(('localhost', 8801)).AndReturn(sock)
sock.getsockname().AndReturn(('127.0.0.1', 8801))
eventlet.spawn_n(eventlet.backdoor.backdoor_server,
sock,
locals=mox.IsA(dict))
self.mox.ReplayAll()
svc = service.Service()
launcher = service.launch(svc)
self.assertEqual(svc.backdoor_port, 8801)
launcher.stop()
def start(self):
super(Service, self).start()
transport = messaging.get_transport(self.conf)
self.sysinv_conductor = messaging.RPCClient(
transport,
messaging.Target(
topic=constants.SYSINV_CONDUCTOR_TOPIC))
self.ceph_api = wrapper.CephWrapper(
endpoint='https://localhost:5001/')
# Get initial config from sysinv and send it to
# services that need it before starting them
self.rpc_server = messaging.get_rpc_server(
transport,
messaging.Target(topic=constants.CEPH_MANAGER_TOPIC,
server=self.conf.sysinv_api_bind_ip),
[RpcEndpoint(self)],
executor='eventlet')
self.rpc_server.start()
eventlet.spawn_n(self.monitor.run)
def start(self):
if self.daemon:
fstools.do_daemon(self.uid, self.gid, self.docroot, self.pidfile)
else:
fstools.do_foreground(self.uid, self.gid, self.docroot)
self.status = True
try:
eventlet.spawn_n(self.handle_events)
except:
self.status = False
raise
try:
self.log.info("http - start %s" % str(self.addr))
self.sock = eventlet.listen(self.addr)
except (KeyboardInterrupt, eventlet.StopServe):
self.status = False
raise
server(self.sock, self.handle, protocol=HttpProtocol, log=self.log)
self.status = False
def notify(action, alarm, state, reason):
LOG.info("Notifying alarm %s in state %s with action %s because %s",
alarm, state, action, reason)
body = {'alarm': alarm, 'state': state, 'reason': reason}
kwargs = {'data': jsonutils.dumps(body)}
if action.scheme == 'https':
default_verify = int(cfg.CONF.alarm.rest_notifier_ssl_verify)
options = urlparse.parse_qs(action.query)
verify = bool(
int(
options.get('ceilometer-alarm-ssl-verify',
[default_verify])[-1]))
kwargs['verify'] = verify
cert = cfg.CONF.alarm.rest_notifier_certificate_file
key = cfg.CONF.alarm.rest_notifier_certificate_key
if cert:
kwargs['cert'] = (cert, key) if key else cert
eventlet.spawn_n(requests.post, action.geturl(), **kwargs)
def __init__(self, side, ttime=2):
forward, reverse, pwm = _PINS[side]
self._pin_forward = forward
self._pin_reverse = reverse
self._pin_pwm = pwm
self._current = 0
self._target = 0
self._delay = ttime / 200.0
self._step = 200.0 / ttime
self._thread = eventlet.spawn_n(self._update)
self._finished = False
self._start()
def plyr_start(self, rq):
server = rq['data']['server']
port = rq['data']['port']
rs = False
if (self.tcpCon != None):
return
try:
print('===>CLIENT MODE initiated!:[', server, ':', port, ']<===')
self.tcpCon = socket.socket()
self.tcpCon.connect((server, port))
self.tcpCon.setblocking(0)
self.tcpCon.settimeout(.5)
eventlet.spawn_n(waitRoomMsgNode, self.tcpCon)
rs = True
except Exception as e:
self.tcpCon = None
print('===>CLIENT MODE initiated failed to connect<===')
rs = False
#js_snd({'do':'plyr_start','data':rs})
self.plyr_loadState(None)
def _interrupt_with_output(self, readable):
"""
For each of the readable file descriptors, find all greenlets which were not themselves
selecting but were interested in the output, and spawn a greenlet to go wake each
of them up.
"""
greenlets_set = set()
for fd in readable:
shell_instance = self._shells_by_fds.get(fd)
if not shell_instance:
LOG.error(
"Shell for readable file descriptor '%d' is missing" %
(fd, ))
else:
greenlets_to_notify = self._greenlets_to_notify.get(
shell_instance.pid, [])
greenlets_set.update(greenlets_to_notify)
nsi = NewShellInterrupt([])
for greenlet_to_notify in greenlets_set:
eventlet.spawn_n(self._interrupt_conditionally, greenlet_to_notify,
nsi)
def _work(self):
"""Process the information regarding the available ports."""
if self._refresh_cache:
# Inconsistent cache might cause exceptions. For example,
# if a port has been removed, it will be known in the next
# loop. Using the old switch port can cause exceptions.
LOG.debug("Refreshing os_win caches...")
self._utils.update_cache()
self._refresh_cache = False
if self._bound_ports or self._unbound_ports:
eventlet.spawn_n(self._notify_plugin_on_port_updates)
# notify plugin about port deltas
if self._added_ports:
LOG.debug("Agent loop has new devices!")
self._treat_devices_added()
if self._removed_ports:
LOG.debug("Agent loop has lost devices...")
self._treat_devices_removed()
def _handle_periodic(self):
"""
Called every second. Kills shells which haven't been asked about in conf.SHELL_TIMEOUT
seconds (currently 600).
"""
try:
keys_to_pop = []
current_time = time.time()
for key, shell_instance in self._shells.iteritems():
if shell_instance.last_output_sent or shell_instance.remove_at_next_iteration:
keys_to_pop.append(key)
else:
difftime = current_time - shell_instance.time_received
if difftime >= shell.conf.SHELL_TIMEOUT.get():
keys_to_pop.append(key)
removed_pids = [self._shells.get(key).pid for key in keys_to_pop]
for key in keys_to_pop:
self._cleanup_shell(key)
finally:
eventlet.spawn_n(self._cleanup_greenlets_for_removed_pids, removed_pids)
eventlet.spawn_after(1, self._handle_periodic)
def send_ip_addr_adv_notif(ns_name, iface_name, address, count=3):
"""Send advance notification of an IP address assignment.
If the address is in the IPv4 family, send gratuitous ARP.
If the address is in the IPv6 family, no advance notification is
necessary, since the Neighbor Discovery Protocol (NDP), Duplicate
Address Discovery (DAD), and (for stateless addresses) router
advertisements (RAs) are sufficient for address resolution and
duplicate address detection.
:param ns_name: Namespace name which GARPs are gonna be sent from.
:param iface_name: Name of interface which GARPs are gonna be sent from.
:param address: Advertised IP address.
:param count: (Optional) How many GARPs are gonna be sent. Default is 3.
"""
def arping():
_arping(ns_name, iface_name, address, count)
if count > 0 and netaddr.IPAddress(address).version == 4:
eventlet.spawn_n(arping)
def echo_udp():
"""
spawn a udp server, bind to a local port,
receive data, echo data back to client,
return port the socket is receiving on
"""
usock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
usock.bind(('0.0.0.0', 0))
port = usock.getsockname()[1]
def serve():
while True:
data, addr = usock.recvfrom(2048)
if not data:
break
usock.sendto(data, addr)
eventlet.spawn_n(serve)
eventlet.sleep(0)
return port
def spawn_n(func, *args, **kwargs):
"""See spawn() above"""
profiler_info = _collect_profiler_info()
@functools.wraps(func)
def wrapper(*args, **kwargs):
if profiler_info:
profiler.init(**profiler_info)
return func(*args, **kwargs)
return eventlet.spawn_n(wrapper, *args, **kwargs)
def starmap(self, function, iterable):
"""This is the same as :func:`itertools.starmap`, except that *func* is
executed in a separate green thread for each item, with the concurrency
limited by the pool's size. In operation, starmap consumes a constant
amount of memory, proportional to the size of the pool, and is thus
suited for iterating over extremely long input lists.
"""
if function is None:
function = lambda *a: a
# We use a whole separate greenthread so its spawn() calls can block
# without blocking OUR caller. On the other hand, we must assume that
# our caller will immediately start trying to iterate over whatever we
# return. If that were a GreenPile, our caller would always see an
# empty sequence because the hub hasn't even entered _do_map() yet --
# _do_map() hasn't had a chance to spawn a single greenthread on this
# GreenPool! A GreenMap is safe to use with different producer and
# consumer greenthreads, because it doesn't raise StopIteration until
# the producer has explicitly called done_spawning().
gi = GreenMap(self.size)
eventlet.spawn_n(self._do_map, function, iterable, gi)
return gi
def spawn_n(func, *args, **kwargs):
"""Passthrough method for eventlet.spawn_n.
This utility exists so that it can be stubbed for testing without
interfering with the service spawns.
It will also grab the context from the threadlocal store and add it to
the store on the new thread. This allows for continuity in logging the
context when using this method to spawn a new thread.
"""
_context = common_context.get_current()
@functools.wraps(func)
def context_wrapper(*args, **kwargs):
# NOTE: If update_store is not called after spawn_n it won't be
# available for the logger to pull from threadlocal storage.
if _context is not None:
_context.update_store()
func(*args, **kwargs)
eventlet.spawn_n(context_wrapper, *args, **kwargs)
def put(self, metadata):
if self._extension == '.ts':
metadata['deleted'] = True
self._sync_buffer()
while self._buffer:
self._sync_buffer()
# zero index, chunk-count is len
metadata['X-Kinetic-Chunk-Count'] = self._chunk_id
metadata['X-Kinetic-Chunk-Nounce'] = self._nounce
metadata['name'] = self._name
self._metadata = metadata
blob = msgpack.packb(self._metadata)
timestamp = diskfile.Timestamp(metadata['X-Timestamp'])
key = self.object_key(timestamp.internal, self._extension,
self._nounce)
self._submit_write(key, blob, final=True)
self._wait_write()
if self.unlink_wait:
self._unlink_old(timestamp)
else:
spawn_n(self._unlink_old, timestamp)
def inspect_hardware(self, task):
"""Inspect hardware to obtain the hardware properties.
This particular implementation only starts inspection using
ironic-discoverd. Results will be checked in a periodic task.
:param task: a task from TaskManager.
:returns: states.INSPECTING
"""
LOG.debug(
'Starting inspection for node %(uuid)s using '
'ironic-discoverd client %(version)s', {
'uuid': task.node.uuid,
'version': ironic_discoverd.__version__
})
# NOTE(dtantsur): we're spawning a short-living green thread so that
# we can release a lock as soon as possible and allow ironic-discoverd
# to operate on a node.
eventlet.spawn_n(_start_inspection, task.node.uuid, task.context)
return states.INSPECTING
def _multi_send(method,
context,
topic,
msg,
timeout=None,
envelope=False,
_msg_id=None,
allowed_remote_exmods=None):
"""Wraps the sending of messages.
Dispatches to the matchmaker and sends message to all relevant hosts.
"""
allowed_remote_exmods = allowed_remote_exmods or []
conf = CONF
LOG.debug(' '.join(map(pformat, (topic, msg))))
queues = _get_matchmaker().queues(topic)
LOG.debug("Sending message(s) to: %s", queues)
# Don't stack if we have no matchmaker results
if not queues:
LOG.warn(_("No matchmaker results. Not casting."))
# While not strictly a timeout, callers know how to handle
# this exception and a timeout isn't too big a lie.
raise rpc_common.Timeout(_("No match from matchmaker."))
# This supports brokerless fanout (addresses > 1)
return_val = None
for queue in queues:
_topic, ip_addr = queue
_addr = "tcp://%s:%s" % (ip_addr, conf.rpc_zmq_port)
if method.__name__ == '_cast':
eventlet.spawn_n(method, _addr, context, _topic, msg, timeout,
envelope, _msg_id)
else:
return_val = method(_addr, context, _topic, msg, timeout, envelope,
allowed_remote_exmods)
return return_val
def _handle_message(self, message):
if message.command == "JOIN":
self.logger.info("Dispatching JOIN handlers...")
for func in self.on_join:
sender = message.sender
channel = Channel(message.params[0])
eventlet.spawn_n(
func,
JoinEvent(self, sender, channel)
)
elif message.command == "PRIVMSG":
for handler in self.on_privmsg:
eventlet.spawn_n(
handler,
PrivmsgEvent(self, message)
)
message_split = message.params[1].split()
if message_split[0] in (self.nickname, self.nickname + "!") and \
len(message_split) > 1:
command = message_split[1]
params = message_split[2:]
self.logger.info("Got command '%s' with params %s",
command, params)
if command in self.commands:
eventlet.spawn_n(
self.commands[command],
CommandEvent(self, message, params)
)
def _make_app_iter(self, node, source, response):
"""
Returns an iterator over the contents of the source (via its read
func). There is also quite a bit of cleanup to ensure garbage
collection works and the underlying socket of the source is closed.
:param response: The webob.Response object this iterator should be
assigned to via response.app_iter.
:param source: The httplib.Response object this iterator should read
from.
:param node: The node the source is reading from, for logging purposes.
"""
try:
try:
# Spawn reader to read from the source and place in the queue.
# We then drop any reference to the source or node, for garbage
# collection purposes.
queue = Queue(1)
spawn_n(self._make_app_iter_reader, node, source, queue,
self.app.logger.thread_locals)
source = node = None
while True:
chunk = queue.get(timeout=self.app.node_timeout)
if isinstance(chunk, bool): # terminator
success = chunk
if not success:
raise Exception(
_('Failed to read all data'
' from the source'))
break
yield chunk
except Empty:
raise ChunkReadTimeout()
except (GeneratorExit, Timeout):
self.app.logger.warn(_('Client disconnected on read'))
except Exception:
self.app.logger.exception(_('Trying to send to client'))
raise
finally:
response.app_iter = None
def _execute_exec_api_rows(self, rows):
def exec_api(session, kwargs):
LOG.info("Making API request %s.", kwargs)
try:
session.request(**kwargs)
except Exception:
LOG.exception('Exception in making API request %s.', kwargs)
for row in rows:
(endpoint, path, method, body, parameters, headers) = row
if endpoint in self._exec_api_endpoints:
kwargs = {
'endpoint_override': self._exec_api_endpoints[endpoint],
'url': path,
'method': method.upper(),
'connect_retries': 10,
'status_code_retries': 10
}
body = json.loads(body)
if body is not None:
kwargs['json'] = body
parameters = json.loads(parameters)
if parameters is not None:
kwargs['params'] = parameters
headers = json.loads(headers)
if headers is not None:
kwargs['headers'] = headers
if cfg.CONF.enable_execute_action:
eventlet.spawn_n(exec_api,
self._exec_api_sessions[endpoint], kwargs)
else:
LOG.info("Simulating API request %s", kwargs)
else:
LOG.warning(
'No configured API endpoint with name %s. '
'Skipping the API request: '
'(endpoint, path, method, body, parameters, headers) '
'= %s.', endpoint, row)
eventlet.sleep(0) # defer to greenthreads running api requests
def test_closure(self):
def spam_to_me(address):
sock = eventlet.connect(address)
while True:
try:
sock.sendall(b'hello world')
# Arbitrary delay to not use all available CPU, keeps the test
# running quickly and reliably under a second
time.sleep(0.001)
except socket.error as e:
if get_errno(e) == errno.EPIPE:
return
raise
server = eventlet.listen(('127.0.0.1', 0))
sender = eventlet.spawn(spam_to_me, server.getsockname())
client, address = server.accept()
server.close()
def reader():
try:
while True:
data = client.recv(1024)
assert data
# Arbitrary delay to not use all available CPU, keeps the test
# running quickly and reliably under a second
time.sleep(0.001)
except socket.error as e:
# we get an EBADF because client is closed in the same process
# (but a different greenthread)
if get_errno(e) != errno.EBADF:
raise
def closer():
client.close()
reader = eventlet.spawn(reader)
eventlet.spawn_n(closer)
reader.wait()
sender.wait()
def test_connect_tcp(self):
def accept_once(listenfd):
try:
conn, addr = listenfd.accept()
fd = conn.makefile(mode='wb')
conn.close()
fd.write(b'hello\n')
fd.close()
finally:
listenfd.close()
server = eventlet.listen(('0.0.0.0', 0))
eventlet.spawn_n(accept_once, server)
client = eventlet.connect(('127.0.0.1', server.getsockname()[1]))
fd = client.makefile('rb')
client.close()
eq_(fd.readline(), b'hello\n')
eq_(fd.read(), b'')
fd.close()
check_hub()
def start(self, timeout=5.0):
"""
循环启动
"""
timeout = float(timeout)
self.running = True
# 孵化心跳循环线程
eventlet.spawn_n(self.heart_beat_loop)
eventlet.sleep(0.1)
overtime = monotonic() + timeout
while True:
if self.connection_pool._created_connections > 0:
break
if monotonic() > overtime:
LOG.error('Redis connection pool init fail')
self.running = False
# 等待前面的绿色线程结束
# eventlet.sleep(1.0)
raise ConnectionError('redis connection pool empty over %1.2f seconds' % timeout)
eventlet.sleep(timeout/5.0)
# 孵化垃圾key删除循环
eventlet.spawn_n(self.garbage_collector_loop)
def api_create():
"""
La función del API para crear elecciones.
Acepta una petición POST que envíe un JSON con formato:
::
{
name: ...,
start_time: ...,
end_time: ...,
public_key: ...,
voter_list: ...,
option_list: ...
}
Con estos datos, crea el blockchain correspondiente, lo distribuye
entre los nodos conocidos y empieza a computar la tabla de
descifrado para usarla al final de la votación.
"""
election_data = request.get_json()
name = election_data.get("name")
election_id = SystemRandom().randint(0, 2**256 - 1)
while election_id in chain_ring:
election_id = SystemRandom().randint(0, 2**256 - 1)
start_time = float(election_data.get("start_time", time.time()))
end_time = float(election_data.get("end_time", time.time() + 3600))
public_key = election_data.get("public_key")
voter_list = election_data.get("voter_list")
option_list = election_data.get("option_list")
chain_ring[election_id] = Blockchain(start_time, None, end_time,
reconstruct_key(public_key),
voter_list, option_list, name)
save_chain_ring(chain_ring, CHAIN_RING_FILE)
eventlet.spawn_n(update_decryption_table, decrypt_table,
chain_ring[election_id].public_key)
logger.info("Elección creada: " + name)
return jsonify({election_id: chain_ring[election_id].serialize()})
def setUp(self):
super(TestMonitorDaemon, self).setUp()
bridge = self.useFixture(net_helpers.OVSBridgeFixture()).bridge
self.machines = self.useFixture(mf.PeerMachines(bridge))
self.router, self.peer = self.machines.machines[:2]
conf_dir = self.get_default_temp_dir().path
monitor = keepalived_state_change.MonitorDaemon(
self.get_temp_file_path('monitor.pid'),
uuidutils.generate_uuid(),
1,
2,
self.router.namespace,
conf_dir,
'foo-iface',
self.machines.ip_cidr
)
eventlet.spawn_n(monitor.run, run_as_root=True)
monitor_started = functools.partial(
lambda mon: mon.monitor is not None, monitor)
utils.wait_until_true(monitor_started)
self.addCleanup(monitor.monitor.stop)
def initialize_if_enabled():
backdoor_locals = {
'exit': _dont_use_this, # So we don't exit the entire process
'quit': _dont_use_this, # So we don't exit the entire process
'fo': _find_objects,
'pgt': _print_greenthreads,
'pnt': _print_nativethreads,
}
if CONF.backdoor_port is None:
return None
start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
# NOTE(johannes): The standard sys.displayhook will print the value of
# the last expression and set it to __builtin__._, which overwrites
# the __builtin__._ that gettext sets. Let's switch to using pprint
# since it won't interact poorly with gettext, and it's easier to
# read the output too.
def displayhook(val):
if val is not None:
pprint.pprint(val)
sys.displayhook = displayhook
sock = _listen('localhost', start_port, end_port, eventlet.listen)
# In the case of backdoor port being zero, a port number is assigned by
# listen(). In any case, pull the port number out here.
port = sock.getsockname()[1]
LOG.info(
_LI('Eventlet backdoor listening on %(port)s for process %(pid)d') % {
'port': port,
'pid': os.getpid()
})
eventlet.spawn_n(eventlet.backdoor.backdoor_server,
sock,
locals=backdoor_locals)
return port
def send_garp_for_proxyarp(ns_name, iface_name, address, count):
"""
Send a gratuitous arp using given namespace, interface, and address
This version should be used when proxy arp is in use since the interface
won't actually have the address configured. We actually need to configure
the address on the interface and then remove it when the proxy arp has been
sent.
"""
def arping_with_temporary_address():
# Configure the address on the interface
device = IPDevice(iface_name, namespace=ns_name)
net = netaddr.IPNetwork(str(address))
device.addr.add(str(net))
_arping(ns_name, iface_name, address, count)
# Delete the address from the interface
device.addr.delete(str(net))
if count > 0:
eventlet.spawn_n(arping_with_temporary_address)
