def __init__(self,
connection=None,
id=None,
name=None,
exchange=None,
logger=None,
agent=None,
outbox_exchange=None,
group_exchange=None,
**kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.outbox_exchange = outbox_exchange or self.outbox_exchange
self.agent = agent
if self.default_fields is None:
self.default_fields = {}
# - setup exchanges and queues
self.exchange = exchange or self.get_direct_exchange()
if group_exchange:
self._scatter_exchange = Exchange(group_exchange,
'fanout',
auto_delete=True)
typemap = {
ACTOR_TYPE.DIRECT: [self.get_direct_queue, self._inbox_direct],
ACTOR_TYPE.RR: [self.get_rr_queue, self._inbox_rr],
ACTOR_TYPE.SCATTER: [self.get_scatter_queue, self._inbox_scatter]
}
self.type_to_queue = {k: v[0] for k, v in items(typemap)}
self.type_to_exchange = {k: v[1] for k, v in items(typemap)}
if not self.outbox_exchange:
self.outbox_exchange = Exchange(
'cl.%s.output' % self.name,
type='topic',
)
# - setup logging
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.id))
self.log = Log('!<%s>' % logger_name, logger=logger)
self.state = self.contribute_to_state(self.construct_state())
# actor specific initialization.
self.construct()
def to_rabbitmq_queue_arguments(arguments, **options):
# type: (Mapping, **Any) -> Dict
"""Convert queue arguments to RabbitMQ queue arguments.
This is the implementation for Channel.prepare_queue_arguments
for AMQP-based transports. It's used by both the pyamqp and librabbitmq
transports.
Arguments:
arguments (Mapping):
User-supplied arguments (``Queue.queue_arguments``).
Keyword Arguments:
expires (float): Queue expiry time in seconds.
This will be converted to ``x-expires`` in int milliseconds.
message_ttl (float): Message TTL in seconds.
This will be converted to ``x-message-ttl`` in int milliseconds.
max_length (int): Max queue length (in number of messages).
This will be converted to ``x-max-length`` int.
max_length_bytes (int): Max queue size in bytes.
This will be converted to ``x-max-length-bytes`` int.
max_priority (int): Max priority steps for queue.
This will be converted to ``x-max-priority`` int.
Returns:
Dict: RabbitMQ compatible queue arguments.
"""
prepared = dictfilter(
dict(
_to_rabbitmq_queue_argument(key, value)
for key, value in items(options)))
return dict(arguments, **prepared) if prepared else arguments
def to_rabbitmq_queue_arguments(arguments, **options):
# type: (Mapping, **Any) -> Dict
"""Convert queue arguments to RabbitMQ queue arguments.
This is the implementation for Channel.prepare_queue_arguments
for AMQP-based transports. It's used by both the pyamqp and librabbitmq
transports.
Arguments:
arguments (Mapping):
User-supplied arguments (``Queue.queue_arguments``).
Keyword Arguments:
expires (float): Queue expiry time in seconds.
This will be converted to ``x-expires`` in int milliseconds.
message_ttl (float): Message TTL in seconds.
This will be converted to ``x-message-ttl`` in int milliseconds.
max_length (int): Max queue length (in number of messages).
This will be converted to ``x-max-length`` int.
max_length_bytes (int): Max queue size in bytes.
This will be converted to ``x-max-length-bytes`` int.
max_priority (int): Max priority steps for queue.
This will be converted to ``x-max-priority`` int.
Returns:
Dict: RabbitMQ compatible queue arguments.
"""
prepared = dictfilter(dict(
_to_rabbitmq_queue_argument(key, value)
for key, value in items(options)
))
return dict(arguments, **prepared) if prepared else arguments
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.hostname == "localhost":
conninfo.hostname = "127.0.0.1"
opts = dict(
{
"host": conninfo.host,
"userid": conninfo.userid,
"password": conninfo.password,
"login_method": conninfo.login_method,
"virtual_host": conninfo.virtual_host,
"insist": conninfo.insist,
"ssl": conninfo.ssl,
"connect_timeout": conninfo.connect_timeout,
"heartbeat": conninfo.heartbeat,
},
**conninfo.transport_options or {}
)
conn = self.Connection(**opts)
conn.client = self.client
return conn
def __init__(
self,
url,
method="GET",
on_ready=None,
on_timeout=None,
on_stream=None,
on_prepare=None,
on_header=None,
headers=None,
**kwargs
):
self.url = url
self.method = method or self.method
self.on_ready = maybe_promise(on_ready) or promise()
self.on_timeout = maybe_promise(on_timeout)
self.on_stream = maybe_promise(on_stream)
self.on_prepare = maybe_promise(on_prepare)
self.on_header = maybe_promise(on_header)
if kwargs:
for k, v in items(kwargs):
setattr(self, k, v)
if not isinstance(headers, Headers):
headers = Headers(headers or {})
self.headers = headers
def assertDictContainsSubset(self, expected, actual, msg=None):
missing, mismatched = [], []
for key, value in items(expected):
if key not in actual:
missing.append(key)
elif value != actual[key]:
mismatched.append('%s, expected: %s, actual: %s' % (
safe_repr(key), safe_repr(value),
safe_repr(actual[key])))
if not (missing or mismatched):
return
standard_msg = ''
if missing:
standard_msg = 'Missing: %s' % ','.join(map(safe_repr, missing))
if mismatched:
if standard_msg:
standard_msg += '; '
standard_msg += 'Mismatched values: %s' % (
','.join(mismatched))
self.fail(self._formatMessage(msg, standard_msg))
def _wait_multiple(self, channels, allowed_methods, timeout=None):
for channel_id, channel in items(channels):
method_queue = channel.method_queue
for queued_method in method_queue:
method_sig = queued_method[0]
if (allowed_methods is None or method_sig in allowed_methods
or method_sig == (20, 40)):
method_queue.remove(queued_method)
method_sig, args, content = queued_method
return channel_id, method_sig, args, content
# Nothing queued, need to wait for a method from the peer
read_timeout = self.read_timeout
wait = self.wait
while 1:
channel, method_sig, args, content = read_timeout(timeout)
if (channel in channels and allowed_methods is None
or method_sig in allowed_methods
or method_sig == (20, 40)):
return channel, method_sig, args, content
# Not the channel and/or method we were looking for. Queue
# this method for later
channels[channel].method_queue.append((method_sig, args, content))
#
# If we just queued up a method for channel 0 (the Connection
# itself) it's probably a close method in reaction to some
# error, so deal with it right away.
#
if channel == 0:
wait()
def __init__(self, connection, **kwargs):
self.connection = connection
self._consumers = set()
self._cycle = None
self._tag_to_queue = {}
self._active_queues = []
self._qos = None
self.closed = False
# instantiate exchange types
self.exchange_types = {
typ: cls(self) for typ, cls in items(self.exchange_types)
}
try:
self.channel_id = self.connection._avail_channel_ids.pop()
except IndexError:
raise ResourceError(
'No free channel ids, current={0}, channel_max={1}'.format(
len(self.connection.channels),
self.connection.channel_max), (20, 10),
)
topts = self.connection.client.transport_options
for opt_name in self.from_transport_options:
try:
setattr(self, opt_name, topts[opt_name])
except KeyError:
pass
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.hostname == 'localhost':
conninfo.hostname = '127.0.0.1'
if conninfo.ssl and 'server_hostname' not in conninfo.ssl:
conninfo.ssl['server_hostname'] = conninfo.hostname
opts = dict(
{
'host': conninfo.host,
'userid': conninfo.userid,
'password': conninfo.password,
'login_method': conninfo.login_method,
'virtual_host': conninfo.virtual_host,
'insist': conninfo.insist,
'ssl': conninfo.ssl,
'connect_timeout': conninfo.connect_timeout,
'heartbeat': conninfo.heartbeat,
}, **conninfo.transport_options or {})
conn = self.Connection(**opts)
conn.client = self.client
conn.connect()
return conn
def _wait_multiple(self, channels, allowed_methods, timeout=None):
for channel_id, channel in items(channels):
method_queue = channel.method_queue
for queued_method in method_queue:
method_sig = queued_method[0]
if (allowed_methods is None
or method_sig in allowed_methods
or method_sig == (20, 40)):
method_queue.remove(queued_method)
method_sig, args, content = queued_method
return channel_id, method_sig, args, content
# Nothing queued, need to wait for a method from the peer
read_timeout = self.read_timeout
wait = self.wait
while 1:
channel, method_sig, args, content = read_timeout(timeout)
if (channel in channels
and allowed_methods is None
or method_sig in allowed_methods
or method_sig == (20, 40)):
return channel, method_sig, args, content
# Not the channel and/or method we were looking for. Queue
# this method for later
channels[channel].method_queue.append((method_sig, args, content))
#
# If we just queued up a method for channel 0 (the Connection
# itself) it's probably a close method in reaction to some
# error, so deal with it right away.
#
if channel == 0:
wait()
def __init__(self, connection, **kwargs):
self.connection = connection
self._consumers = set()
self._cycle = None
self._tag_to_queue = {}
self._active_queues = []
self._qos = None
self.closed = False
# instantiate exchange types
self.exchange_types = dict(
(typ, cls(self)) for typ, cls in items(self.exchange_types))
try:
self.channel_id = self.connection._avail_channel_ids.pop()
except IndexError:
raise ResourceError(
'No free channel ids, current={0}, channel_max={1}'.format(
len(self.connection.channels),
self.connection.channel_max),
(20, 10),
)
topts = self.connection.client.transport_options
for opt_name in self.from_transport_options:
try:
setattr(self, opt_name, topts[opt_name])
except KeyError:
pass
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.ssl:
raise NotImplementedError(NO_SSL_ERROR)
opts = dict(
{
"host": conninfo.host,
"userid": conninfo.userid,
"password": conninfo.password,
"virtual_host": conninfo.virtual_host,
"login_method": conninfo.login_method,
"insist": conninfo.insist,
"ssl": conninfo.ssl,
"connect_timeout": conninfo.connect_timeout,
},
**conninfo.transport_options or {}
)
conn = self.Connection(**opts)
conn.client = self.client
self.client.drain_events = conn.drain_events
return conn
def kwdict(kwargs): # pragma: no cover # noqa
"""Make sure keyword arguments are not in Unicode.
This should be fixed in newer Python versions,
see: http://bugs.python.org/issue4978.
"""
return dict((key.encode("utf-8"), value) for key, value in items(kwargs))
def test_parse_generated_as_uri(self):
conn = Connection(self.url)
info = conn.info()
for k, v in items(self.expected):
self.assertEqual(info[k], v)
# by default almost the same- no password
self.assertEqual(conn.as_uri(), self.nopass)
self.assertEqual(conn.as_uri(include_password=True), self.url)
def can(self, actor):
able = set()
for id, state in items(self.agents):
if actor in state['actors']:
# remove the . from the agent, which means that the
# agent is a clone of another agent.
able.add(id.partition('.')[0])
return able
def can(self, actor):
able = set()
for id, state in items(self.agents):
if actor in state['actors']:
# remove the . from the agent, which means that the
# agent is a clone of another agent.
able.add(id.partition('.')[0])
return able
def test_parse_generated_as_uri(self):
conn = Connection(self.url)
info = conn.info()
for k, v in items(self.expected):
self.assertEqual(info[k], v)
# by default almost the same- no password
self.assertEqual(conn.as_uri(), self.nopass)
self.assertEqual(conn.as_uri(include_password=True), self.url)
def kwdict(kwargs): # pragma: no cover # noqa
"""Make sure keyword arguments are not in Unicode.
This should be fixed in newer Python versions,
see: http://bugs.python.org/issue4978.
"""
return dict(
(key.encode('utf-8'), value) for key, value in items(kwargs))
def update_strategies(self):
loader = self.app.loader
for name, task in items(self.app.tasks):
self.strategies[name] = task.start_strategy(self.app, self)
task.__trace__ = build_tracer(name,
task,
loader,
self.hostname,
app=self.app)
def expire_agents(self):
expired = set()
for id, state in items(self._agents):
if state and state.get('ts'):
if time() > state['ts'] + self.heartbeat_expire:
expired.add(id)
for id in expired:
self._remove_agent(id)
return self._agents
def expire_agents(self):
expired = set()
for id, state in items(self._agents):
if state and state.get('ts'):
if time() > state['ts'] + self.heartbeat_expire:
expired.add(id)
for id in expired:
self._remove_agent(id)
return self._agents
def request(self, method, path, body=None, headers=None):
self.path = path
self.method = method
if body is not None:
try:
read = body.read
except AttributeError:
self.body = body
else:
self.body = read()
if headers is not None:
self.headers.extend(list(items(headers)))
def request(self, method, path, body=None, headers=None):
self.path = path
self.method = method
if body is not None:
try:
read = body.read
except AttributeError:
self.body = body
else:
self.body = read()
if headers is not None:
self.headers.extend(list(items(headers)))
def __init__(self, connection=None, id=None, name=None, exchange=None,
logger=None, agent=None, outbox_exchange=None,
group_exchange=None, **kwargs):
self.connection = connection
self.id = id or uuid()
self.name = name or self.name or self.__class__.__name__
self.outbox_exchange = outbox_exchange or self.outbox_exchange
self.agent = agent
if self.default_fields is None:
self.default_fields = {}
# - setup exchanges and queues
self.exchange = exchange or self.get_direct_exchange()
if group_exchange:
self._scatter_exchange = Exchange(
group_exchange, 'fanout', auto_delete=True)
typemap = {
ACTOR_TYPE.DIRECT: [self.get_direct_queue, self._inbox_direct],
ACTOR_TYPE.RR: [self.get_rr_queue, self._inbox_rr],
ACTOR_TYPE.SCATTER: [self.get_scatter_queue, self._inbox_scatter]
}
self.type_to_queue = {k: v[0] for k, v in items(typemap)}
self.type_to_exchange = {k: v[1] for k, v in items(typemap)}
if not self.outbox_exchange:
self.outbox_exchange = Exchange(
'cl.%s.output' % self.name, type='topic',
)
# - setup logging
logger_name = self.name
if self.agent:
logger_name = '%s#%s' % (self.name, shortuuid(self.id))
self.log = Log('!<%s>' % logger_name, logger=logger)
self.state = self.contribute_to_state(self.construct_state())
# actor specific initialization.
self.construct()
def __wrapped(*args, **kwargs):
info = ''
if self.ident:
info += self.ident.format(self.instance)
info += '{0.__name__}('.format(meth)
if args:
info += ', '.join(map(repr, args))
if kwargs:
if args:
info += ', '
info += ', '.join('{k}={v!r}'.format(k=key, v=value)
for key, value in items(kwargs))
info += ')'
self.logger.debug(info)
return meth(*args, **kwargs)
def __wrapped(*args, **kwargs):
info = ''
if self.ident:
info += self.ident.format(self.instance)
info += '{0.__name__}('.format(meth)
if args:
info += ', '.join(map(repr, args))
if kwargs:
if args:
info += ', '
info += ', '.join('{k}={v!r}'.format(k=key, v=value)
for key, value in items(kwargs))
info += ')'
self.logger.debug(info)
return meth(*args, **kwargs)
def __init__(self, url, method='GET', on_ready=None, on_timeout=None,
on_stream=None, on_prepare=None, on_header=None,
headers=None, **kwargs):
self.url = url
self.method = method or self.method
self.on_ready = maybe_promise(on_ready) or promise()
self.on_timeout = maybe_promise(on_timeout)
self.on_stream = maybe_promise(on_stream)
self.on_prepare = maybe_promise(on_prepare)
self.on_header = maybe_promise(on_header)
if kwargs:
for k, v in items(kwargs):
setattr(self, k, v)
if not isinstance(headers, Headers):
headers = Headers(headers or {})
self.headers = headers
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
conn = self.Connection(host=conninfo.host,
userid=conninfo.userid,
password=conninfo.password,
virtual_host=conninfo.virtual_host,
login_method=conninfo.login_method,
insist=conninfo.insist,
ssl=conninfo.ssl,
connect_timeout=conninfo.connect_timeout)
conn.client = self.client
self.client.drain_events = conn.drain_events
return conn
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.hostname == 'localhost':
conninfo.hostname = '127.0.0.1'
conn = self.Connection(host=conninfo.host,
userid=conninfo.userid,
password=conninfo.password,
login_method=conninfo.login_method,
virtual_host=conninfo.virtual_host,
insist=conninfo.insist,
ssl=conninfo.ssl,
connect_timeout=conninfo.connect_timeout)
conn.client = self.client
return conn
def start(self, c):
info("mingle: searching for neighbors")
I = c.app.control.inspect(timeout=1.0, connection=c.connection)
replies = I.hello(c.hostname, revoked._data) or {}
replies.pop(c.hostname, None)
if replies:
info("mingle: sync with %s nodes", len([reply for reply, value in items(replies) if value]))
for reply in values(replies):
if reply:
try:
other_clock, other_revoked = MINGLE_GET_FIELDS(reply)
except KeyError: # reply from pre-3.1 worker
pass
else:
c.app.clock.adjust(other_clock)
revoked.update(other_revoked)
info("mingle: sync complete")
else:
info("mingle: all alone")
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.hostname == 'localhost':
conninfo.hostname = '127.0.0.1'
conn = self.Connection(host=conninfo.host,
userid=conninfo.userid,
password=conninfo.password,
login_method=conninfo.login_method,
virtual_host=conninfo.virtual_host,
insist=conninfo.insist,
ssl=conninfo.ssl,
connect_timeout=conninfo.connect_timeout,
heartbeat=conninfo.heartbeat)
conn.client = self.client
return conn
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.ssl:
raise NotImplementedError(NO_SSL_ERROR)
conn = self.Connection(host=conninfo.host,
userid=conninfo.userid,
password=conninfo.password,
virtual_host=conninfo.virtual_host,
login_method=conninfo.login_method,
insist=conninfo.insist,
ssl=conninfo.ssl,
connect_timeout=conninfo.connect_timeout)
conn.client = self.client
self.client.drain_events = conn.drain_events
return conn
def start(self, c):
info('mingle: searching for neighbors')
I = c.app.control.inspect(timeout=1.0, connection=c.connection)
replies = I.hello(c.hostname, revoked._data) or {}
replies.pop(c.hostname, None)
if replies:
info('mingle: sync with %s nodes',
len([reply for reply, value in items(replies) if value]))
for reply in values(replies):
if reply:
try:
other_clock, other_revoked = MINGLE_GET_FIELDS(reply)
except KeyError: # reply from pre-3.1 worker
pass
else:
c.app.clock.adjust(other_clock)
revoked.update(other_revoked)
info('mingle: sync complete')
else:
info('mingle: all alone')
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.ssl:
raise NotImplementedError(NO_SSL_ERROR)
opts = dict({
'host': conninfo.host,
'userid': conninfo.userid,
'password': conninfo.password,
'virtual_host': conninfo.virtual_host,
'login_method': conninfo.login_method,
'insist': conninfo.insist,
'ssl': conninfo.ssl,
'connect_timeout': conninfo.connect_timeout,
}, **conninfo.transport_options or {})
conn = self.Connection(**opts)
conn.client = self.client
self.client.drain_events = conn.drain_events
return conn
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.hostname == 'localhost':
conninfo.hostname = '127.0.0.1'
opts = dict({
'host': conninfo.host,
'userid': conninfo.userid,
'password': conninfo.password,
'login_method': conninfo.login_method,
'virtual_host': conninfo.virtual_host,
'insist': conninfo.insist,
'ssl': conninfo.ssl,
'connect_timeout': conninfo.connect_timeout,
'heartbeat': conninfo.heartbeat,
}, **conninfo.transport_options or {})
conn = self.Connection(**opts)
conn.client = self.client
return conn
def establish_connection(self):
"""Establish connection to the AMQP broker."""
conninfo = self.client
for name, default_value in items(self.default_connection_params):
if not getattr(conninfo, name, None):
setattr(conninfo, name, default_value)
if conninfo.hostname == "localhost":
conninfo.hostname = "127.0.0.1"
opts = dict(
{
"host": conninfo.host,
"userid": conninfo.userid,
"password": conninfo.password,
"login_method": conninfo.login_method,
"virtual_host": conninfo.virtual_host,
"insist": conninfo.insist,
"ssl": conninfo.ssl,
"connect_timeout": conninfo.connect_timeout,
"heartbeat": conninfo.heartbeat,
}, **conninfo.transport_options or {})
conn = self.Connection(**opts)
conn.client = self.client
conn.connect()
return conn
def assertDictContainsSubset(self, expected, actual, msg=None):
missing, mismatched = [], []
for key, value in items(expected):
if key not in actual:
missing.append(key)
elif value != actual[key]:
mismatched.append(
'%s, expected: %s, actual: %s' %
(safe_repr(key), safe_repr(value), safe_repr(actual[key])))
if not (missing or mismatched):
return
standard_msg = ''
if missing:
standard_msg = 'Missing: %s' % ','.join(map(safe_repr, missing))
if mismatched:
if standard_msg:
standard_msg += '; '
standard_msg += 'Mismatched values: %s' % (','.join(mismatched))
self.fail(self._formatMessage(msg, standard_msg))
def repr_writers(h):
"""Return description of pending writers."""
return ['({0}){1}->{2}'.format(fd, _rcb(cb), repr_flag(WRITE))
for fd, cb in items(h.writers)]
def _setup_request(self, curl, request, buffer, headers, _pycurl=pycurl):
setopt = curl.setopt
setopt(_pycurl.URL, bytes_to_str(request.url))
# see tornado curl client
request.headers.setdefault('Expect', '')
request.headers.setdefault('Pragma', '')
setopt(
_pycurl.HTTPHEADER,
['{0}: {1}'.format(*h) for h in items(request.headers)],
)
setopt(
_pycurl.HEADERFUNCTION,
partial(request.on_header or self.on_header, request.headers),
)
setopt(
_pycurl.WRITEFUNCTION,
request.on_stream or buffer.write,
)
setopt(
_pycurl.FOLLOWLOCATION,
request.follow_redirects,
)
setopt(
_pycurl.USERAGENT,
bytes_to_str(request.user_agent or DEFAULT_USER_AGENT),
)
if request.network_interface:
setopt(_pycurl.INTERFACE, request.network_interface)
setopt(
_pycurl.ENCODING,
'gzip,deflate' if request.use_gzip else 'none',
)
if request.proxy_host:
if not request.proxy_port:
raise ValueError('Request with proxy_host but no proxy_port')
setopt(_pycurl.PROXY, request.proxy_host)
setopt(_pycurl.PROXYPORT, request.proxy_port)
if request.proxy_username:
setopt(
_pycurl.PROXYUSERPWD,
'{0}:{1}'.format(request.proxy_username,
request.proxy_password or ''))
else:
setopt(_pycurl.PROXY, '')
curl.unsetopt(_pycurl.PROXYUSERPWD)
setopt(_pycurl.SSL_VERIFYPEER, 1 if request.validate_cert else 0)
setopt(_pycurl.SSL_VERIFYHOST, 2 if request.validate_cert else 0)
if request.ca_certs is not None:
setopt(_pycurl.CAINFO, request.ca_certs)
setopt(_pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)
for meth in METH_TO_CURL.values():
setopt(meth, False)
try:
meth = METH_TO_CURL[request.method]
except KeyError:
curl.setopt(_pycurl.CUSTOMREQUEST, request.method)
else:
curl.unsetopt(_pycurl.CUSTOMREQUEST)
setopt(meth, True)
if request.method in ('POST', 'PUT'):
body = request.body.encode('utf-8') if request.body else bytes()
reqbuffer = BytesIO(body)
setopt(_pycurl.READFUNCTION, reqbuffer.read)
if request.method == 'POST':
def ioctl(cmd):
if cmd == _pycurl.IOCMD_RESTARTREAD:
reqbuffer.seek(0)
setopt(_pycurl.IOCTLFUNCTION, ioctl)
setopt(_pycurl.POSTFIELDSIZE, len(body))
else:
setopt(_pycurl.INFILESIZE, len(body))
elif request.method == 'GET':
assert not request.body
if request.auth_username is not None:
auth_mode = {
'basic': _pycurl.HTTPAUTH_BASIC,
'digest': _pycurl.HTTPAUTH_DIGEST
}[request.auth_mode or 'basic']
setopt(_pycurl.HTTPAUTH, auth_mode)
userpwd = '{0}:{1}'.format(
request.auth_username,
request.auth_password or '',
)
setopt(_pycurl.USERPWD, userpwd)
else:
curl.unsetopt(_pycurl.USERPWD)
if request.client_cert is not None:
setopt(_pycurl.SSLCERT, request.client_cert)
if request.client_key is not None:
setopt(_pycurl.SSLKEY, request.client_key)
if request.on_prepare is not None:
request.on_prepare(curl)
def assert_info(self, conn, **fields):
info = conn.info()
for field, expected in items(fields):
assert info[field] == expected
def dictfilter(d=None, **kw):
"""Remove all keys from dict ``d`` whose value is :const:`None`."""
d = kw if d is None else (dict(d, **kw) if kw else d)
return {k: v for k, v in items(d) if v is not None}
def assert_info(self, conn, **fields):
info = conn.info()
for field, expected in items(fields):
self.assertEqual(info[field], expected)
def update_strategies(self):
loader = self.app.loader
for name, task in items(self.app.tasks):
self.strategies[name] = task.start_strategy(self.app, self)
task.__trace__ = build_tracer(name, task, loader, self.hostname,
app=self.app)
def repr_readers(h):
return ['({0}){1}->{2}'.format(fd, _rcb(cb), repr_flag(READ | ERR))
for fd, cb in items(h.readers)]
def dictfilter(d=None, **kw):
"""Remove all keys from dict ``d`` whose value is :const:`None`"""
d = kw if d is None else (dict(d, **kw) if kw else d)
return {k: v for k, v in items(d) if v is not None}
def reset_rate_limits(self):
self.task_buckets.update(
(n, self.bucket_for_task(t)) for n, t in items(self.app.tasks))
def repr_writers(h):
return ['({0}){1}->{2}'.format(fd, _rcb(cb), repr_flag(WRITE))
for fd, cb in items(h.writers)]
def assert_info(self, conn, **fields):
info = conn.info()
for field, expected in items(fields):
self.assertEqual(info[field], expected)
def reprkwargs(kwargs, sep=', ', fmt='{0}={1}'):
return sep.join(fmt.format(k, _safe_repr(v)) for k, v in items(kwargs))
def getheaders(self):
return list(items(self.response.headers))
def repr_readers(h):
"""Return description of pending readers."""
return ['({0}){1}->{2}'.format(fd, _rcb(cb), repr_flag(READ | ERR))
for fd, cb in items(h.readers)]
def test_items(self):
c = LRUCache()
c.update(a=1, b=2, c=3)
assert list(items(c))
def _repr_writers(self):
return ['({0}){1}->{2}'.format(fd, _rcb(cb), repr_flag(WRITE))
for fd, cb in items(self.writers)]
def getheaders(self):
return list(items(self.response.headers))
def reprkwargs(kwargs, sep=", ", fmt="{0}={1}"):
return sep.join(fmt.format(k, _safe_repr(v)) for k, v in items(kwargs))
def reprkwargs(kwargs, sep=', ', fmt='{0}={1}'):
return sep.join(fmt.format(k, _safe_repr(v)) for k, v in items(kwargs))
def test_items(self):
c = LRUCache()
c.update(a=1, b=2, c=3)
self.assertTrue(list(items(c)))
def _setup_request(self, curl, request, buffer, headers, _pycurl=pycurl):
setopt = curl.setopt
setopt(_pycurl.URL, bytes_to_str(request.url))
# see tornado curl client
request.headers.setdefault('Expect', '')
request.headers.setdefault('Pragma', '')
setopt(
_pycurl.HTTPHEADER,
['{0}: {1}'.format(*h) for h in items(request.headers)],
)
setopt(
_pycurl.HEADERFUNCTION,
partial(request.on_header or self.on_header, request.headers),
)
setopt(
_pycurl.WRITEFUNCTION, request.on_stream or buffer.write,
)
setopt(
_pycurl.FOLLOWLOCATION, request.follow_redirects,
)
setopt(
_pycurl.USERAGENT,
bytes_to_str(request.user_agent or DEFAULT_USER_AGENT),
)
if request.network_interface:
setopt(_pycurl.INTERFACE, request.network_interface)
setopt(
_pycurl.ENCODING, 'gzip,deflate' if request.use_gzip else 'none',
)
if request.proxy_host:
if not request.proxy_port:
raise ValueError('Request with proxy_host but no proxy_port')
setopt(_pycurl.PROXY, request.proxy_host)
setopt(_pycurl.PROXYPORT, request.proxy_port)
if request.proxy_username:
setopt(_pycurl.PROXYUSERPWD, '{0}:{1}'.format(
request.proxy_username, request.proxy_password or ''))
else:
setopt(_pycurl.PROXY, '')
curl.unsetopt(_pycurl.PROXYUSERPWD)
setopt(_pycurl.SSL_VERIFYPEER, 1 if request.validate_cert else 0)
setopt(_pycurl.SSL_VERIFYHOST, 2 if request.validate_cert else 0)
if request.ca_certs is not None:
setopt(_pycurl.CAINFO, request.ca_certs)
setopt(_pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)
for meth in METH_TO_CURL.values():
setopt(meth, False)
try:
meth = METH_TO_CURL[request.method]
except KeyError:
curl.setopt(_pycurl.CUSTOMREQUEST, request.method)
else:
curl.unsetopt(_pycurl.CUSTOMREQUEST)
setopt(meth, True)
if request.method in ('POST', 'PUT'):
body = request.body or ''
reqbuffer = BytesIO(body)
setopt(_pycurl.READFUNCTION, reqbuffer.read)
if request.method == 'POST':
def ioctl(cmd):
if cmd == _pycurl.IOCMD_RESTARTREAD:
reqbuffer.seek(0)
setopt(_pycurl.IOCTLFUNCTION, ioctl)
setopt(_pycurl.POSTFIELDSIZE, len(body))
else:
setopt(_pycurl.INFILESIZE, len(body))
elif request.method == 'GET':
assert not request.body
if request.auth_username is not None:
auth_mode = {
'basic': _pycurl.HTTPAUTH_BASIC,
'digest': _pycurl.HTTPAUTH_DIGEST
}[request.auth_mode or 'basic']
setopt(_pycurl.HTTPAUTH, auth_mode)
userpwd = '{0}:{1}'.format(
request.auth_username, request.auth_password or '',
)
setopt(_pycurl.USERPWD, userpwd)
else:
curl.unsetopt(_pycurl.USERPWD)
if request.client_cert is not None:
setopt(_pycurl.SSLCERT, request.client_cert)
if request.client_key is not None:
setopt(_pycurl.SSLKEY, request.client_key)
if request.on_prepare is not None:
request.on_prepare(curl)
def _repr_readers(self):
return ['({0}){1}->{2}'.format(fd, _rcb(cb), repr_flag(READ | ERR))
for fd, cb in items(self.readers)]
def reset_rate_limits(self):
self.task_buckets.update(
(n, self.bucket_for_task(t)) for n, t in items(self.app.tasks)
)
