def test_with_Greenlet(self):
rl = local.local()
rl.foo = 1
def root():
l = MyLocal(42)
assert l
# And an empty local.
l2 = local.local()
assert l2
gevent.getcurrent().spawn_tree_locals['a value'] = 42
io = NativeStrIO()
g = gevent.spawn(util.print_run_info, file=io)
g.join()
return io.getvalue()
g = gevent.spawn(root)
g.name = 'Printer'
g.join()
value = g.value
self.assertIn("Spawned at", value)
self.assertIn("Parent:", value)
self.assertIn("Spawn Tree Locals", value)
self.assertIn("Greenlet Locals:", value)
self.assertIn('MyLocal', value)
self.assertIn("Printer", value) # The name is printed
# Empty locals should not be printed
self.assertNotIn('{}', value)
def test_with_Greenlet(self):
rl = local.local()
rl.foo = 1
def root():
l = MyLocal(42)
assert l
# And an empty local.
l2 = local.local()
assert l2
gevent.getcurrent().spawn_tree_locals['a value'] = 42
io = NativeStrIO()
g = gevent.spawn(util.print_run_info, file=io)
g.join()
return io.getvalue()
g = gevent.spawn(root)
g.name = 'Printer'
g.join()
value = g.value
self.assertIn("Spawned at", value)
self.assertIn("Parent:", value)
self.assertIn("Spawn Tree Locals", value)
self.assertIn("Greenlet Locals:", value)
self.assertIn('MyLocal', value)
self.assertIn("Printer", value) # The name is printed
# Empty locals should not be printed
self.assertNotIn('{}', value)
def __init__(self, token, client=None):
super(APIClient, self).__init__()
self.client = client
self.http = HTTPClient(token, self._after_requests)
self._captures = local()
def __init__(self, socket_factory, pool_size):
self.pid = os.getpid()
self.pool_size = pool_size
self.socket_factory = socket_factory
self.sockets = Queue()
self.log = logging.getLogger('ming.async.AsyncPool')
self.local = local()
def session_manager(environ):
global gRequests, gRequest
if gRequests is None:
gRequests = local()
gRequest = LocalProxy(lambda: gRequests.request)
gRequests.request = Request(environ)
yield
gRequests.request = None
def root():
l = MyLocal(42)
assert l
# And an empty local.
l2 = local.local()
assert l2
gevent.getcurrent().spawn_tree_locals['a value'] = 42
io = NativeStrIO()
g = gevent.spawn(util.print_run_info, file=io)
g.join()
return io.getvalue()
def setup_env():
class env: pass
env.config = readconf.parse_config()
use_gevent = not env.config.threaded
localprov = setup_process_model(use_gevent)
commonlib.helpers.random_key_gen = commonlib.helpers.RandomKeyFactory(env.config.random_str)
env.mailer = commonlib.messaging.email.Mailer(env.config.mail)
env.mailer.start()
env.context = localprov.local()
commonlib.helpers.push_to_builtins('env', env)
return env
def root():
l = MyLocal(42)
assert l
# And an empty local.
l2 = local.local()
assert l2
gevent.getcurrent().spawn_tree_locals['a value'] = 42
io = NativeStrIO()
g = gevent.spawn(util.print_run_info, file=io)
g.join()
return io.getvalue()
def __init__(self, managers):
self.managers = managers
self.c = MessageCrypto()
self.lock = Semaphore()
self.__players = []
self.__cfg = self.managers.Database().Config()
# Init greenlet local
self.session = local()
self.session.player = None
def __init__(self, engine, min_sessions=10, max_sessions=25):
self.engine = engine
sm = sessionmaker(autoflush=False, autocommit=False, bind=engine)
self.session = scoped_session(sm)
self.min_sessions = min_sessions
self.max_sessions = max_sessions
self.session_pool = []
self.available = []
self.checkouts = {}
self.sessions = local()
self.lock = Semaphore()
def __init__(self, engine, min_sessions=10, max_sessions=25):
self.engine = engine
sm = sessionmaker(autoflush=False, autocommit=False, bind=engine)
self.session = scoped_session(sm)
self.min_sessions = min_sessions
self.max_sessions = max_sessions
self.session_pool = []
self.available = []
self.checkouts = {}
self.sessions = local()
self.lock = Semaphore()
def setup_env(conf):
class env: pass
try:
conf_local = __import__(conf)
except Exception as err:
print(err)
conf_local = None
env.config = readconf.parse_config(conf_default, conf_local)
commonlib.helpers.random_key_gen = commonlib.helpers.RandomKeyFactory(env.config.random_str)
#env.mailer = commonlib.messaging.email.Mailer(env.config.mail)
#env.mailer.start()
env.context = localprov.local()
builtins.env = env
return env
class ThreadSafeHttp(object):
__scoped_credentials = None
__local = local()
def __init__(self, creds):
self.__scoped_credentials = Connection._create_scoped_credentials(
creds, Connection.SCOPE)
def __getattr__(self, name):
if not hasattr(self.__local, 'http'):
self.__local.http = self.__scoped_credentials.authorize(Http())
return getattr(self.__local.http, name)
def __init__(self, dbapimodule, **kwargs):
thread_name = threading.currentThread().getName()
if callable(dbapimodule):
_log.debug("Constructing Driver for %s in thread: %s",
dbapimodule.__name__, thread_name)
connect = dbapimodule
else:
_log.debug("Constructing Driver for %s in thread: %s", dbapimodule,
thread_name)
_log.debug("kwargs for connect is %r", kwargs)
dbapimodule = importlib.import_module(dbapimodule)
connect = lambda: dbapimodule.connect(**kwargs)
self.__connect = connect
self.__connection = None
self.stash = local()
class ThreadSafeHttp(object):
__scoped_credentials = None
__local = local()
def __init__(self, creds):
self.__scoped_credentials = with_scopes_if_required(
creds, Connection.SCOPE)
def __getattr__(self, name):
if not hasattr(self.__local, 'http'):
self.__local.http = google_auth_httplib2.AuthorizedHttp(
self.__scoped_credentials)
return getattr(self.__local.http, name)
class ContextFilter(object):
"""
Context filter
DOC : https://docs.python.org/3/howto/logging-cookbook.html#adding-contextual-information-to-your-logging-output
"""
# CAUTION : We must target local here as global (otherwise its a variable local to functions)
LOC = local()
@classmethod
def set_value(cls, k, v):
"""
Set context value
:param k: key name
:type k: basestring
:param v: value
:type v: object
"""
Assert.check(Exception, k is not None, "Need k, got None")
Assert.check(Exception, len(k) > 0, "Need k, got empty")
setattr(cls.LOC, k, v)
def filter(self, record):
"""
Record filter.
This will push thread context (using LOC) toward logger item "kfilter", as an OrderedDict, formatted as "key0:value0 keyN:valueN"
:param record: logging.LogRecord
:type record: logging.LogRecord
:return: bool
:rtype bool
"""
# Prepare
s = u""
for k in sorted(self.LOC.__dict__.keys()):
v = self.LOC.__dict__[k]
s += u" %s:%s" % (k, v)
s += u" "
# Push to record in a single shot
setattr(record, "kfilter", s)
return True
def init_app(routes=DEFAULT_ROUTES):
import logging
from loki import errors
from loki.utils import encode_json, capture_exception
from loki.utils.gevent import register_hub_error_handler
assert getattr(
init_app, 'init_flag',
False) is False, "loki.app.init_app should not be called twice"
app.setup()
# Change settings from ENV
if os.environ.get('LOKI_DEBUG'):
logging.info('settings %s changed from ENV: -> %s', 'DEBUG', True)
app.settings['DEBUG'] = True
app.register_json_encoder(encode_json)
app.route_many(routes)
# In tornado
@app.register_application_configurator
def config_sentry(application):
application.sentry_client = sentry_client
# In gevent
@register_hub_error_handler
def gevent_error_handler(context, exc_info):
e = exc_info[1]
if isinstance(e, errors.SentryRequestFailed):
logging.warn('Send exception to sentry failed: %s', e)
else:
capture_exception(exc_info=exc_info)
global g
g = local.local()
app._init_application()
if settings.DEBUG:
app.log_app_info()
init_app.init_flag = True
def test_with_Greenlet(self):
rl = local.local()
rl.foo = 1
def root():
l = MyLocal(42)
assert l
gevent.getcurrent().spawn_tree_locals['a value'] = 42
g = gevent.spawn(util.format_run_info)
g.join()
return g.value
g = gevent.spawn(root)
g.name = 'Printer'
g.join()
value = '\n'.join(g.value)
self.assertIn("Spawned at", value)
self.assertIn("Parent:", value)
self.assertIn("Spawn Tree Locals", value)
self.assertIn("Greenlet Locals:", value)
self.assertIn('MyLocal', value)
self.assertIn("Printer", value) # The name is printed
def test_local(self):
"""
线程局部变量
很多集成了gevent的web框架将HTTP会话对象以线程局部变量的方式存储在gevent内
"""
stash = local()
def f1():
stash.x = 1
print(stash.x)
def f2():
stash.y = 2
print(stash.y)
try:
stash.x
except AttributeError:
print("x is not local to f2")
g1 = gevent.spawn(f1)
g2 = gevent.spawn(f2)
gevent.joinall([g1, g2])
def test_local(self):
"""
线程局部变量
很多集成了gevent的web框架将HTTP会话对象以线程局部变量的方式存储在gevent内
"""
stash = local()
def f1():
stash.x = 1
print(stash.x)
def f2():
stash.y = 2
print(stash.y)
try:
stash.x
except AttributeError:
print("x is not local to f2")
g1 = gevent.spawn(f1)
g2 = gevent.spawn(f2)
gevent.joinall([g1, g2])
# -*-coding:utf-8-*-
import gevent
from gevent.local import local
local_data = local()
def f1():
local_data.x = 1
print(f'({gevent.getcurrent()}) set value: {local_data.x}')
def f2():
local_data.y = 2
print(f'({gevent.getcurrent()}) set value: {local_data.y}')
try:
gevent.sleep(1)
local_data.x
except AttributeError:
print('x is not local to f2')
if __name__ == '__main__':
gevent.joinall([
gevent.spawn(f1),
gevent.spawn(f2),
])
def __delitem__(self, item):
del self.__dict__[item]
def update(self, dict):
self.__dict__.update(dict)
def clean(self):
self.__dict__ = {}
def replace(self, dict):
self.__dict__ = dict
# default variables
envs = local()
envs.common = AttributedDict(
{
'interactive': True,
'show_errors': False,
'parallel': False,
'ask_passwd': False,
'functions': {},
'localhost': [
'localhost',
'127.0.0.1',
gethostname(),
],
'split_function': ':',
'split_args': ',',
sem = BoundedSemaphore(2)
def worker(n):
sem.acquire()
print('Worker %i acquired semaphore' % n)
gevent.sleep(0)
sem.release()
print('Worker %i released semaphore' % n)
gevent.joinall([gevent.spawn(worker, i) for i in xrange(0, 6)])
#同线程类似，协程也有本地变量，也就是只在当前协程内可被访问的变量
from gevent.local import local
data = local()
def f1():
data.x = 1
print data.x
def f2():
try:
print data.x
except AttributeError:
print "x is not visible"
gevent.joinall([gevent.spawn(f1), gevent.spawn(f2)])
# -- code --
import db.models # noqa
@instantiate
class DBState(object):
__slots__ = (
'engine',
'session_maker',
)
Session = None
current = local()
def init(connstr, drop_first=False):
global Session
engine = create_engine(
connstr,
encoding='utf-8',
convert_unicode=True,
echo=False,
isolation_level='SERIALIZABLE',
)
drop_first and Model.metadata.drop_all(engine)
Model.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
self.exception = e
self.results[node] = e
if len(list(self.results.values())) == self.callNum:
self.ready = True
self.asyncResult.set()
def set(self, node, msg):
self.results[node] = msg
if len(list(self.results.values())) == self.callNum:
self.ready = True
self.asyncResult.set()
local_func_call_context = local()
class Controller(Greenlet):
def __init__(self, dl=None, ul=None):
Greenlet.__init__(self)
self.log = logging.getLogger("{module}.{name}".format(
module=self.__class__.__module__, name=self.__class__.__name__))
self.uuid = str(uuid.uuid4())
self.name = "Controller"
self.info = "WiSHFUL COntroller"
self.config = None
self.default_callback = None
self.callbacks = {}
self.call_id_gen = 0
# coding: utf-8
"""
hub.py
``````
realize a hub greenlets :_request_ctx_hub: for request context
greenlet switching.
:copyright: (c) 2016 by neo1218.
:license: MIT, see LICENSE for more details.
"""
# from gevent.wsgi import WSGIServer
from gevent.local import local
from werkzeug.local import LocalProxy
_request_ctx_hub = local()
request = LocalProxy(lambda: _request_ctx_hub.request)
current_app = LocalProxy(lambda: _request_ctx_hub.app)
from gevent.local import local
# Import from itools
from itools.database.fields import get_field_and_datatype
from itools.datatypes import String
from itools.log import Logger
from itools.validators import ValidationError
# Local imports
from exceptions import FormError
###########################################################################
# Keep the context globally
###########################################################################
g = local()
def set_context(ctx):
if ctx and get_context() != None:
raise ValueError('Cannot set context')
g.context = ctx
def get_context():
return getattr(g, 'context', None)
#######################################################################
# Internationalization
#######################################################################
def __init__(self):
self.locals = local()
self._clients = []
self._muted_clients = []
self.broadcast_users_count(self.BROADCAST_USERS_COUNT_INTERVAL)
def __init__(self):
self.stash = local()
self.exception = e
self.results[node] = e
if len(list(self.results.values())) == self.callNum:
self.ready = True
self.asyncResult.set()
def set(self, node, msg):
self.results[node] = msg
if len(list(self.results.values())) == self.callNum:
self.ready = True
self.asyncResult.set()
local_func_call_context = local()
class Controller(Greenlet):
def __init__(self, dl=None, ul=None):
Greenlet.__init__(self)
self.log = logging.getLogger("{module}.{name}".format(
module=self.__class__.__module__, name=self.__class__.__name__))
self.uuid = str(uuid.uuid4())
self.name = "Controller"
self.info = "WiSHFUL COntroller"
self.config = None
self.default_callback = None
self.callbacks = {}
def __init__(self):
self.stash = local()
import random
import gevent
from gevent.queue import Queue, Empty
from gevent.pool import Group
from gevent import getcurrent
from gevent.local import local
taskqueue = Queue(maxsize=5)
workerdata = local()
def worker(pid):
print 'worker #%d-%s starting' % (pid,id(getcurrent()))
workerdata.cnt = 0
while True:
try:
task = taskqueue.get(timeout=5)
workerdata.cnt += 1
print 'worker #%d got task #%d' % (pid,task)
gevent.sleep(random.randint(0,3))
except Empty:
break
print 'worker #%d-%s exiting (cnt:%d)' % (pid,id(getcurrent()),workerdata.cnt)
def boss():
print 'boss starting'
for i in xrange(0,15):
taskqueue.put(i)
print 'boss completed'
print('Assigned all work in iteration 2')
gevent.joinall([
gevent.spawn(boss),
gevent.spawn(worker, 'steve'),
gevent.spawn(worker, 'john'),
gevent.spawn(worker, 'bob'),
])
#给gevent上下文来指定那些数据是本地的。
import gevent
from gevent.local import local
stash = local()
def f1():
stash.x = 1
print(stash.x)
def f2():
stash.y = 2
print(stash.y)
try:
stash.x
except AttributeError:
print("x is not local to f2")
g1 = gevent.spawn(f1)
import bottle
from gevent.local import local
from sakura.common.bottle import PicklableFileRequest
from sakura.common.errors import APIRequestErrorOfflineDatastore
from sakura.common.tools import ObservableEvent
from sakura.hub.secrets import TemporarySecretsRegistry
from sakura.hub.web.transfers import Transfer
# object storing greenlet-local data
greenlet_env = local()
def get_context():
return HubContext._instance
class HubContext(object):
_instance = None
PW_RECOVERY_SECRETS_LIFETIME = 10 * 60
class global_events:
on_datastores_change = ObservableEvent()
def __init__(self, db, planner):
self.db = db
self.planner = planner
self.daemons = self.db.Daemon
self.dataflows = self.db.Dataflow
self.users = self.db.User
self.sessions = self.db.Session
self.op_classes = self.db.OpClass
self.op_instances = self.db.OpInstance
self.links = self.db.Link
self.op_params = self.db.OpParam
self.datastores = self.db.Datastore
from gevent.local import local
from werkzeug.local import LocalProxy
from werkzeug.wrappers import Request
from contextlib import contextmanager
from gevent.wsgi import WSGIServer
_requests = local()
request = LocalProxy(lambda: _requests.request)
@contextmanager
def sessionmanager(environ):
_requests.request = Request(environ)
yield
_requests.request = None
def logic():
return "Hello " + request.remote_addr
def application(environ, start_response):
status = '200 OK'
with sessionmanager(environ):
body = logic()
headers = [
('Content-Type', 'text/html')
]
start_response(status, headers)
return [body]
#-*- coding:utf8 -*-
__author__ = 'admin'
# Gevent也允许你指定局部于greenlet上下文的数据。
# 在内部，它被实现为以greenlet的getcurrent()为键，
# 在一个私有命名空间寻址的全局查找。
import gevent
from gevent.local import local
stash = local()
def f1():
stash.x = 1
print(stash.x)
def f2():
stash.y = 2
print(stash.y)
try:
stash.x
except AttributeError:
print('x is not local to f2')
g1 = gevent.spawn(f1)
g2 = gevent.spawn(f2)
gevent.joinall([g1,g2])
# 1
# 2
# x is not local to f2
logging.Formatter(
'%(asctime)s %(levelname)s %(module)s:%(lineno)s %(message)s'))
_handler.setFormatter(
logging.Formatter(
'%(asctime)s %(module)s %(levelname)s:%(lineno)s %(message)s'))
log = logging.getLogger(name)
log.addHandler(_handler)
log.setLevel(logging.DEBUG)
return log
#设定对共享资源的访问数量
sem = BoundedSemaphore(1)
log = getLogger(__name__)
signal_stop = False
local_stash = local()
evt = Event()
art = AsyncResult()
q = Queue()
# ------------------------------------------------------------------------
# Queue、spawn
# def init():
# while True:
# if signal_stop: break
# art.set(False)
# gevent.sleep(0.1)
# log.info('init...')
# art.set(True)
# evt.set()
# gevent.sleep(10)
### import
from adict import adict
import gevent
from gevent.local import local
from gevent.queue import Queue, Empty
from gevent.socket import wait_read, wait_write
from psycopg2 import connect, Error, extensions, OperationalError, ProgrammingError
from psycopg2.extras import RealDictCursor
import sys
### init
_db_config = {}
_db_pool = Queue()
_local = local()
_log = None
class db_rollback(Exception):
pass
def _gevent_wait_callback(conn, timeout=None):
# From https://github.com/gevent/gevent/blob/master/examples/psycopg2_pool.py#L19
while 1:
state = conn.poll()
if state == extensions.POLL_OK:
break
elif state == extensions.POLL_READ:
wait_read(conn.fileno(), timeout=timeout)
elif state == extensions.POLL_WRITE:
wait_write(conn.fileno(), timeout=timeout)
from sqlalchemy import create_engine
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.orm import (
Session as _Session,
scoped_session,
sessionmaker,
)
from src.utils.datetime_util import datetime2epoch
from src.utils import gevent_psycopg2
gevent_psycopg2.monkey_patch()
logger = logging.getLogger(__name__)
db_ctx = local.local()
def gen_commit_deco(DBSession):
def wrap(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
register_db_commit = getattr(db_ctx, 'register_db_commit', None)
if not register_db_commit:
db_ctx.register_db_commit = [DBSession]
session = DBSession()
if register_db_commit:
result = func(*args, **kwargs)
return result
try:
result = func(*args, **kwargs)
# Import from gevent
from gevent.local import local
# Import from itools
from itools.database.fields import get_field_and_datatype
from itools.datatypes import String
from itools.log import Logger
from itools.validators import ValidationError
# Local imports
from exceptions import FormError
###########################################################################
# Keep the context globally
###########################################################################
g = local()
def set_context(ctx):
if ctx and get_context() != None:
raise ValueError('Cannot set context')
g.context = ctx
def get_context():
return getattr(g, 'context', None)
#######################################################################
# Internationalization
#######################################################################
def build_call_pattern(method,message=object):
call_pat = CALL_PATTERN.copy()
call_pat['method'] = method
call_pat['message'] = message
return call_pat
def lazy_property(property_name, property_factory, doc=None):
def get(self):
if not hasattr(self, property_name):
setattr(self, property_name, property_factory(self))
return getattr(self, property_name)
return property(get)
_curactor = local.local()
def curactor():
"""Return the current actor."""
return _curactor.current
def _setcurrent(actor):
_curactor.current = actor
def curaddr():
"""Return address of current actor."""
return curactor().address
def curmesh():
def __init__(self):
self.local = local()
self.logger = logging.getLogger('ciserver.GithubWebhook')
# thank you so much Aaron Griffith
from gevent.local import local
import parameterize
# normally parameterize is set to work with threading locals
# we can simulate this in green threads, with a dictionary lookup by
# thread ID
def derp(value):
return list(value.items())
parameterize.set_context_locals(local())
# a coroutine-local-ish stack of contexts
# like implicit arguments passed with every function
# that act like globals
def contextify(klass):
# klass contains defaults, but what actually is here is a parameter
# dict, that you lookup with getattr to get/set current values
# or use call and 'with' to parameterize by copy of current value
d = dict((n,v) for n,v in klass.__dict__.items() if not n.startswith('_'))
p = parameterize.Parameter(d)
class Context:
def __call__(self):
d = dict(p.get())
return p.parameterize(d)
def __getattr__(self,n):
return p.get()[n]
def __setattr__(self,n,v):
import threading
import Queue
import gevent
from gevent import monkey
from gevent import Greenlet
from gevent import pool
from gevent import queue
from gevent import event
from gevent import Timeout
from gevent import threadpool
from gevent.local import local
from gevent.coros import Semaphore
import sys
plist = local()
semaphore = Semaphore()
def work(alist):
semaphore.acquire()
print alist.pop()
semaphore.release()
def thread_test(v):
alist = []
for x in range(11, 20):
alist.append(x)
from gevent.local import local
from collections import OrderedDict
from gevent.event import AsyncResult
from gevent.queue import Queue, Empty
from . import defines as D
from .message import MessageType, Message, Request, Error, Result, BaseMessage
from .errors import ActorError, RequestTimeout
from .service import *
_logger = logging.getLogger(__name__)
# Greenlet-local storage for getting sender's name
__current_actor = local()
def get_current_context():
""" Gets the actor's current context.
:return:
"""
global __current_actor
return getattr(__current_actor, 'context', None)
def set_current_context(ctx):
global __current_actor
__current_actor.context = ctx
def __init__(self):
self._local = local()
self.data = self._local.__dict__
from gevent import monkey
monkey.patch_all() # noqa
from gevent import local
from gevent.pywsgi import WSGIServer
import urllib
import functools
import types
from collections import defaultdict
from functools import partial
from .http_code import http_status as status_code
from .http import Request
urllib.getproxies_environment = lambda: {'_': '_'}
this = local.local()
HTTP_METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'CONNECT']
class MyWebPy(object):
def __init__(self, app_name, headers=None, proxy=None):
"""
Useage:
app = MyWebPy(__name,[
('Content-Type', 'text/html')
])
"""
self.name = app_name
# proxy
def __init__(self):
self.local = local()
def __init__(self):
self.store = local()
self.server_url = 'http://base.icaipiao123.com/input/v1/input/batch'
self.timeout = 5
self.schedule = dict()
self.mongo_client = MongoClient()
import ujson as json
from loki.base.template import TemplateMeta
from loki.job.arguments import TemplateArgument, DeployArgument
from loki.job.base import JobTemplate
from loki.node.nodes import TreeNode
from loki.job.statuses import Status
from loki.job.models import Deployment
from loki.zookeeper import zk
from loki.errors import FatalError
from loki.settings import ZK_JOB_STATUS_PATH
from loki.settings import ZK_NEW_JOB_PATH
from torext.errors import ValidationError
logger = logging.getLogger("loki.job.templates")
greenlet_local = local()
class TemplateMixin(object):
__metaclass__ = TemplateMeta
type = TemplateArgument(type="text", hidden=True, value=lambda self: self.__templatename__)
concurrence_idc = DeployArgument(label=u"机房间是否并发", type="checkbox",required=False)
concurrence_server = DeployArgument(label=u"机器间并发度", type="number", value=1,required=False)
contacters = TemplateArgument(label=u"通知邮箱", type="arrayinput", required=False, value=[])
# required by set_servers_from_node
exclude_offline = TemplateArgument(
label=u'是否排除 traffic_ratio 为 0 机器',
notice=u'若勾选，则发布中 traffic_ratio 为 0 机器不可见',
type="checkbox", value=False)
from bs4 import BeautifulSoup
from gevent.pool import Group
from gevent.event import Event
from gevent.lock import Semaphore
from gevent.local import local
from gevent import Timeout
from ..contrib import contentdisposition
from ..contrib.Win32IconImagePlugin import _accept as is_ico, Win32IcoFile
from .. import plugintools, account, logger, input, interface, settings
from ..config import globalconfig
from ..scheme import transaction
log = logger.get("hoster.util")
localctx = local()
def _get_content_range_bytes(range):
bytes = range.split(' ', 1)[1]
a = bytes.split('-')
b = a[1].split('/')
return int(a[0]), int(b[0]), int(b[1])
def http_response_errors(ctx, resp):
if resp.status_code == 416:
raise plugintools.RangeNotSatisfiedError()
#ctx.fatal('got HTTP 416 - Requested range not satisfiable')
elif resp.status_code == 503:
raise plugintools.NoMoreConnectionsError()
elif resp.status_code == 401:
ctx.fatal('401 Not Authorized')
from utils import instantiate
# -- code --
import db.models # noqa
@instantiate
class DBState(object):
__slots__ = (
'engine',
'session_maker',
)
Session = None
current = local()
def init(connstr, drop_first=False):
global Session
engine = create_engine(
connstr,
encoding='utf-8',
convert_unicode=True,
echo=False,
isolation_level='SERIALIZABLE',
)
drop_first and Model.metadata.drop_all(engine)
Model.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
# encoding:utf-8
"""
http://sdiehl.github.io/gevent-tutorial/
"""
import gevent
from gevent.local import local
from werkzeug.local import LocalProxy
from werkzeug.wrappers import Request
from contextlib import contextmanager
from gevent.wsgi import WSGIServer
# region sample
stash = local()
def f1():
stash.x = 1
print stash.x
def f2():
stash.y = 2
print stash.y
try:
stash.x
except AttributeError:
print 'x is not local to f2'
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
from gevent.local import local
from contextlib import contextmanager
from ..variablesizepool import VariableSizePool
localctx = local()
class host(object):
__slots__ = ["options", "ignore_module", "check_pool", "download_pool"]
def __init__(self, options):
self.options = options()
self.ignore_module = False
self.check_pool = VariableSizePool(size=self.global_max_check_tasks)
self.download_pool = VariableSizePool(size=self.global_max_download_tasks)
def get_account(self, task, file):
return localctx.hoster.get_account(task, file)
@property
def log(self):
import gevent
from gevent.local import local
stash = local()
def f1():
stash.x = 1
print(stash.x)
def f2():
stash.y = 2
print(stash.y)
try:
stash.x
except AttributeError:
print("x is not local to f2")
g1 = gevent.spawn(f1)
g2 = gevent.spawn(f2)
gevent.joinall([g1, g2])
import threading
import Queue
import gevent
from gevent import monkey
from gevent import Greenlet
from gevent import pool
from gevent import queue
from gevent import event
from gevent import Timeout
from gevent import threadpool
from gevent.local import local
from gevent.coros import Semaphore
import sys
plist = local()
semaphore = Semaphore()
def work(alist):
semaphore.acquire()
print alist.pop()
semaphore.release()
def thread_test(v):
alist = []
for x in range(11, 20):
alist.append(x)
pool.map(worker1, xrange(0, 2))
pool.map(worker2, xrange(3, 6))
# 其实不用再多说什么你也应该了解了，当我们使用底层同步元语对象BoundSemaphore(2)
# 初始化两个范围之后，就意味着，同时只能有两个greenlet去拿到这两个锁，在这两个锁被aquire之后，其他人必须等待其被relase，否则无法再继续并发更多的greenlet。上面对比就是个好例子，第一组代码的运行先spawn两个greenlet去拿到锁然后释放锁，这里似乎没有什么不一样。但是work2中你会发现，有锁的话你只能看到3，4
# 一起运行然后再运行5。如果没有sem, 3, 4, 5
# 将会同时运行。而且从代码来看，semaphore还支持上下文管理器，看起来还蛮友好的，更多api和介绍参考文档：http: // www.gevent.org / gevent.lock.html
#
# 3.线程局部变量(ThreadLocals)：
# Gevent也允许你指定局部于greenlet上下文的数据。 在内部，它被实现为以greenlet的getcurrent()
# 为键， 在一个私有命名空间寻址的全局查找。
# 一言不合先上代码：
import gevent
from gevent.local import local
x = local()
def f1():
x.x = 1
print x.x
print x.__dict__
def f2():
x.y = 2
print(x.y)
try:
print x.__dict__
except AttributeError:
#!/usr/bin/env python
from gevent.local import local
from werkzeug.local import LocalProxy
from werkzeug.wrappers import Request
from contextlib import contextmanager
from gevent.wsgi import WSGIServer
_requests = local()
request = LocalProxy(lambda: _requests.request)
@contextmanager
def sessionmanager(environ):
_requests.request = Request(environ)
yield
_requests.request = None
def logic():
return "Hello " + request.remote_addr
def application(environ, start_response):
status = '200 OK'
with sessionmanager(environ):
body = logic()
headers = [('Content-Type', 'text/html')]
global_data[district]["taluka"][taluka]["panchayat"] = data
def fetch_taluka(url, district):
encoded_url = _cleanUrl(url)
data = talukaExtract(encoded_url)
if "taluka" in global_data[district].keys():
global_data[district]["taluka"] =\
dict(data.items() + global_data[district]["taluka"].items())
else:
global_data[district]["taluka"] = data
jobs = [gevent.spawn(fetch_panchayat, value['url'], district, key)
for key, value in data.iteritems()]
gevent.joinall(jobs)
global_data = local.local()
data = districtExtract("http://164.100.112.66/netnrega/writereaddata/citizen"\
"_out/phy_fin_reptemp_Out_18_local_1112.html")
global_data = data
jobs = [gevent.spawn(fetch_taluka,
value["url"],
key)
for key, value in data.iteritems()]
gevent.joinall(jobs)
f = open('database/data.json', 'w')
output = json.dumps(global_data)
print time.time() - start
f.write(output)
def __init__(self):
self.context = local()
super(RPCService, self).__init__(self.name)
for name, method in self.operations.items():
self.mapTarget(remoting.ExtCallableTarget(Wrapper(self, method), name, secure=method.secure))