def bootstrap_pyon(logging_config_override=None, pyon_cfg=None):
"""
This function initializes the core elements of the Pyon framework in a controlled way.
It does not initialize the ION container or the ION system.
@param logging_config_override A dict to initialize the Python logging subsystem (None loads default files)
@param pyon_cfg A DotDict with the fully loaded pyon configuration to merge into CFG (None loads default files)
"""
print "pyon: pyon.bootstrap (bootstrap_pyon) executing..."
# Make sure Pyon is only initialized only once
global pyon_initialized
if pyon_initialized:
print "pyon: WARNING -- bootstrap_pyon() called again!"
return
# ENVIRONMENT. Check we are called in an expected environment (files, directories, etc)
assert_environment()
# LOGGING. Initialize logging from config
if not logutil.is_logging_configured():
logutil.configure_logging(logutil.DEFAULT_LOGGING_PATHS, logging_config_override=logging_config_override)
# YAML patch: OrderedDicts instead of dicts
from pyon.util.yaml_ordered_dict import apply_yaml_patch
apply_yaml_patch()
# CONFIG. Initialize pyon global configuration from local files
set_config(pyon_cfg)
log.debug("pyon: CFG set to %s", CFG)
# OBJECTS. Object and message definitions.
from pyon.core.registry import IonObjectRegistry
global _obj_registry
_obj_registry = IonObjectRegistry()
# SERVICES. Service definitions
# TODO: change the following to read service definitions from directory and import selectively
from pyon.ion.service import IonServiceRegistry
import interface.services
global _service_registry
_service_registry = IonServiceRegistry()
_service_registry.load_service_mods(interface.services)
_service_registry.build_service_map()
# RESOURCES. Load and initialize definitions
from pyon.ion import resource
resource.load_definitions()
# Fix a weird bug on Ubuntu that resets time.sleep to un-monkey patched version on import threading
from gevent import monkey; monkey.patch_time()
# Set initialized flag
pyon_initialized = True
log.debug("pyon: initialized OK")
#this must be in the beginning so things are patched before ever imported by other libraries
from gevent import monkey
# monkey.patch_all()
monkey.patch_socket()
monkey.patch_thread()
monkey.patch_time()
import time
from JumpScale import j
import JumpScale.portal
from JumpScale.baselib.cmdutils import argparse
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--instance', help="Gridportal instance", required=True)
opts = parser.parse_args()
ays =j.atyourservice.get('jumpscale', 'portal', instance=opts.instance)
j.application.instanceconfig = ays.hrd
j.application.start("jumpscale:gridportal")
j.application.initGrid()
j.logger.disable()
j.core.portal.getServer().start()
j.application.stop()
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
#A big thanks to:
# Michael Rose https://bitbucket.org/michaelrose/ganglia-xdr-parser/src/da033e2bed3e/ganglia_xdr_parser.py
# Patrick Debois https://gist.github.com/1376525
from wishbone import Actor
import xdrlib
from time import time
from gevent.monkey import patch_time;patch_time()
class DecodeGangliaException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
class Ganglia(Actor):
'''**DecodeGanglia is MetricFactory module which decodes Ganglia metrics into
a MetricFactory format.***
Receives xdr formatted data coming from a Ganglia client and decodes it
into Wishbone format.
(1381002603.726132, 'hadoop', 'hostname', 'queue.outbox.in_rate', 0, '', ())
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
from wishbone import Actor
from wishbone.errors import QueueLocked
from gevent import monkey;monkey.patch_time()
from time import time
import json
class Elasticsearch(Actor):
'''**Decode Elasticsearch metrics.**
This module takes a JSON formatted coming from an Elasticsearch
clusternode and converts the metrics into the common format.
The name of the metric will be point delimited.
Parameters:
- name (str): The instance name.
def run(self):
"""Initialize the backend and run its main loop."""
self.logger.debug('initializing backend components')
self.logger.debug('initializing cryto provider')
crypto = singleton(CryptoProvider)
pwgen = singleton(PasswordGenerator)
self.logger.debug('initializing database')
fname = os.path.join(self.data_dir, 'bluepass.db')
database = singleton(Database, fname)
database.lock()
self.logger.debug('initializing model')
model = singleton(Model, database)
self.logger.debug('initializing locator')
locator = singleton(Locator)
for ls in platform.get_location_sources():
self.logger.debug('adding location source: {}'.format(ls.name))
locator.add_source(ls())
self.logger.debug('initializing sync API')
listener = util.create_listener(('0.0.0.0', 0))
listener.setblocking(False)
app = singleton(SyncAPIApplication)
syncapi = singleton(SyncAPIServer, listener, app)
syncapi.start()
self.logger.debug('initializing sync API publisher')
publisher = singleton(SyncAPIPublisher, syncapi)
publisher.start()
# The syncer is started at +10 seconds so that the locator will have
# hopefully located all neighbors by then and we can do a once-off
# sync at startup. This is just a heuristic for optimization, the
# correctness of our sync algorithm does not depend on this.
if locator.sources:
self.logger.debug('initializing background sync worker')
syncer = singleton(Syncer)
syncer.start_later(10)
else:
self.logger.warning('no location sources available')
self.logger.warning('network synchronization is disabled')
self.logger.debug('initializing control API')
listener = util.create_listener(self.listen_address)
listener.setblocking(False)
fname = os.path.join(self.data_dir, 'backend.addr')
addr = util.unparse_address(listener.getsockname())
with open(fname, 'w') as fout:
fout.write('{}\n'.format(addr))
self.logger.info('listening on: {}'.format(addr))
handler = SocketAPIHandler()
messagebus = singleton(MessageBusServer, listener, self.auth_token,
handler)
if self.options.get('trace'):
fname = os.path.join(self.data_dir, 'backend.trace')
messagebus.set_trace(fname)
if not self.options.get('daemon'):
def messagebus_event(event, *args):
if event == 'LastConnectionClosed':
self.stop()
messagebus.add_callback(messagebus_event)
messagebus.start()
self.logger.debug('installing signal handlers')
on_signal = get_hub().loop.signal(signal.SIGTERM)
on_signal.start(self.stop)
monkey.patch_time() # Make Thread.join() cooperative.
self.logger.debug('all backend components succesfully initialized')
# This is where the backend runs (until stopped).
self._stop_event.wait()
self.logger.debug('backend event loop terminated')
self.logger.debug('shutting down control API')
messagebus.stop()
self.logger.debug('shutting down database')
database.close()
self.logger.debug('stopped all backend components')
def run(self, app, host=None, port=None, **kwargs):
"""Run the SocketIO web server.
:param app: The Flask application instance.
:param host: The hostname or IP address for the server to listen on.
Defaults to 127.0.0.1.
:param port: The port number for the server to listen on. Defaults to
5000.
:param debug: ``True`` to start the server in debug mode, ``False`` to
start in normal mode.
:param use_reloader: ``True`` to enable the Flask reloader, ``False``
to disable it.
:param extra_files: A list of additional files that the Flask
reloader should watch. Defaults to ``None``
:param log_output: If ``True``, the server logs all incomming
connections. If ``False`` logging is disabled.
Defaults to ``True`` in debug mode, ``False``
in normal mode. Unused when the threading async
mode is used.
:param kwargs: Additional web server options. The web server options
are specific to the server used in each of the supported
async modes. Note that options provided here will
not be seen when using an external web server such
as gunicorn, since this method is not called in that
case.
"""
if host is None:
host = '127.0.0.1'
if port is None:
server_name = app.config['SERVER_NAME']
if server_name and ':' in server_name:
port = int(server_name.rsplit(':', 1)[1])
else:
port = 5000
debug = kwargs.pop('debug', app.debug)
log_output = kwargs.pop('log_output', debug)
use_reloader = kwargs.pop('use_reloader', debug)
extra_files = kwargs.pop('extra_files', None)
app.debug = debug
if app.debug and self.server.eio.async_mode != 'threading':
# put the debug middleware between the SocketIO middleware
# and the Flask application instance
#
# mw1 mw2 mw3 Flask app
# o ---- o ---- o ---- o
# /
# o Flask-SocketIO
# \ middleware
# o
# Flask-SocketIO WebSocket handler
#
# BECOMES
#
# dbg-mw mw1 mw2 mw3 Flask app
# o ---- o ---- o ---- o ---- o
# /
# o Flask-SocketIO
# \ middleware
# o
# Flask-SocketIO WebSocket handler
#
self.sockio_mw.wsgi_app = DebuggedApplication(
self.sockio_mw.wsgi_app, evalex=True)
if self.server.eio.async_mode == 'threading':
from werkzeug._internal import _log
_log(
'warning', 'WebSocket transport not available. Install '
'eventlet or gevent and gevent-websocket for '
'improved performance.')
app.run(host=host,
port=port,
threaded=True,
use_reloader=use_reloader,
**kwargs)
elif self.server.eio.async_mode == 'eventlet':
def run_server():
import eventlet
import eventlet.wsgi
import eventlet.green
addresses = eventlet.green.socket.getaddrinfo(host, port)
if not addresses:
raise RuntimeError(
'Could not resolve host to a valid address')
eventlet_socket = eventlet.listen(addresses[0][4],
addresses[0][0])
# If provided an SSL argument, use an SSL socket
ssl_args = [
'keyfile', 'certfile', 'server_side', 'cert_reqs',
'ssl_version', 'ca_certs', 'do_handshake_on_connect',
'suppress_ragged_eofs', 'ciphers'
]
ssl_params = {k: kwargs[k] for k in kwargs if k in ssl_args}
if len(ssl_params) > 0:
for k in ssl_params:
kwargs.pop(k)
ssl_params['server_side'] = True # Listening requires true
eventlet_socket = eventlet.wrap_ssl(
eventlet_socket, **ssl_params)
eventlet.wsgi.server(eventlet_socket,
app,
log_output=log_output,
**kwargs)
if use_reloader:
run_with_reloader(run_server, extra_files=extra_files)
else:
run_server()
elif self.server.eio.async_mode == 'gevent':
from gevent import pywsgi
try:
from geventwebsocket.handler import WebSocketHandler
websocket = True
except ImportError:
app.logger.warning(
'WebSocket transport not available. Install '
'gevent-websocket for improved performance.')
websocket = False
log = 'default'
if not log_output:
log = None
if websocket:
self.wsgi_server = pywsgi.WSGIServer(
(host, port),
app,
handler_class=WebSocketHandler,
log=log,
**kwargs)
else:
self.wsgi_server = pywsgi.WSGIServer((host, port),
app,
log=log,
**kwargs)
if use_reloader:
# monkey patching is required by the reloader
from gevent import monkey
monkey.patch_thread()
monkey.patch_time()
def run_server():
self.wsgi_server.serve_forever()
run_with_reloader(run_server, extra_files=extra_files)
else:
self.wsgi_server.serve_forever()
"""
Copyright (C) 2019, Zato Source s.r.o. https://zato.io
Licensed under LGPLv3, see LICENSE.txt for terms and conditions.
"""
from __future__ import absolute_import, division, print_function, unicode_literals
# Monkey-patching modules individually can be about 20% faster,
# or, in absolute terms, instead of 275 ms it may take 220 ms.
from gevent.monkey import patch_builtins, patch_contextvars, patch_thread, patch_time, patch_os, patch_queue, patch_select, \
patch_selectors, patch_signal, patch_socket, patch_ssl, patch_subprocess, patch_sys
# Note that the order of patching matters, just like in patch_all
patch_os()
patch_time()
patch_thread()
patch_sys()
patch_socket()
patch_select()
patch_selectors()
patch_ssl()
patch_subprocess()
patch_builtins()
patch_signal()
patch_queue()
patch_contextvars()
# stdlib
import locale
import logging
"""
gevent 协成模块示例
"""
import gevent
# 导入脚本执行time模块操作
from gevent import monkey
monkey.patch_time()
from time import sleep
# 协程函数
def foo(a, b):
print("Running foo ..", a, b)
sleep(3)
print("Foo again")
def bar():
print("Running bar ..")
sleep(2)
print("Bar again")
# 生成协程对象
f = gevent.spawn(foo, 1, 2)
g = gevent.spawn(bar)
gevent.joinall([f, g]) # 阻塞等待f,g执行完
#!/usr/bin/env python # -*- coding: utf-8 -*- # # FileName: run.py # Author : summer1988 ([email protected]) # DateTime: 2015-11-12 17:31 # from gevent.monkey import patch_all, patch_socket, patch_thread, patch_time from pythunder.rpc.servers import FetcherServer # patch_all() patch_thread() patch_socket() patch_time() FetcherServer.serveForever()
# author: Ziming Guo
# time: 2020/4/6
'''
demo03:
gevent 协程模块 示例
'''
import gevent
from gevent import monkey # 首先倒入 monkey
monkey.patch_time() # 在 time 导入之前，执行 patch_time
# 此处应注意，一定是要在 time 倒入之前执行 patch_time
# 之后，time 这个模块所对应的阻塞就会变成导致 gevent 跳转的阻塞
from time import sleep
# 写一个协程函数
def fun01(a, b):
print("Running fun01...", a, b)
sleep(2) # 此时就不用再用 gevent 去调用某个方法了
print("fun01 again...")
def bar():
print("Running bar...")
sleep(3)
print("Bar again...")
