def test_graph(self):
c = HttpClient()
handler = Echo(c._loop)
ws = yield c.get(self.ws_uri, websocket_handler=handler)
self.assertEqual(ws.event('post_request').fired(), 0)
message = yield handler.get()
self.assertTrue(message)
def test_login(self):
http = HttpClient()
url = self.url + self.app.config['LOGIN_URL']
response = yield from http.get(url)
cookie = response.cookies.get('luxtest')
self.assertTrue(cookie)
self.assertEqual(response.status_code, 200)
doc = self.bs(response)
token = self.authenticity_token(doc)
self.assertEqual(len(token), 1)
# try to login
data = {'username': '******', 'password': '******'}
response2 = yield from http.post(url, data=data)
self.assertEqual(response2.status_code, 403)
#
# Add csrf token
data.update(token)
response2 = yield from http.post(url, data=data)
self.assertEqual(response2.status_code, 200)
cookie2 = response2.cookies.get('luxtest')
self.assertTrue(cookie2)
self.assertNotEqual(cookie2.value, cookie.value)
self.assertEqual(response2.headers['content-type'],
'application/json; charset=utf-8')
data = response2.json()
self.assertTrue('redirect' in data)
self.assertEqual(data['success'], True)
#
# Login again should cause MethodNotAllowed
response3 = yield from http.post(url, data=data)
self.assertEqual(response3.status_code, 405)
def setup(self, http=None, timeout=None, **kw):
if not http:
timeout = timeout if timeout is not None else self.default_timeout
http = HttpClient(timeout=timeout, **kw)
http.headers['accept'] = 'application/json, text/*; q=0.5'
http.headers['content-type'] = 'application/json'
self.local.http = http
def __init__(self,
url,
version=None,
data=None,
full_response=False,
http=None,
timeout=None,
sync=False,
loop=None,
encoding='ascii',
**kw):
self.sync = sync
self._url = url
self._version = version or self.__class__.default_version
self._full_response = full_response
self._data = data if data is not None else {}
if not http:
timeout = timeout if timeout is not None else self.default_timeout
if sync and not loop:
loop = new_event_loop()
http = HttpClient(timeout=timeout, loop=loop, **kw)
http.headers['accept'] = 'application/json, text/*; q=0.5'
http.headers['content-type'] = 'application/json'
self._http = http
self._encoding = encoding
def test_graph(self):
c = HttpClient()
handler = Echo(c._loop)
ws = yield c.get(self.ws_uri, websocket_handler=handler)
self.assertEqual(ws.event('post_request').fired(), 0)
message = yield handler.get()
self.assertTrue(message)
def __init__(
self,
url,
version=None,
data=None,
full_response=False,
http=None,
timeout=None,
sync=False,
loop=None,
encoding="ascii",
**kw
):
self.sync = sync
self._url = url
self._version = version or self.__class__.default_version
self._full_response = full_response
self._data = data if data is not None else {}
if not http:
timeout = timeout if timeout is not None else self.default_timeout
if sync and not loop:
loop = new_event_loop()
http = HttpClient(timeout=timeout, loop=loop, **kw)
http.headers["accept"] = "application/json, text/*; q=0.5"
http.headers["content-type"] = "application/json"
self._http = http
self._encoding = encoding
def test_pong(self):
c = HttpClient()
handler = Echo()
ws = yield c.get(self.ws_echo, websocket_handler=handler).on_headers
#
ws.ping('TESTING CLIENT PING')
message = yield handler.get()
self.assertEqual(message, 'PONG: TESTING CLIENT PING')
def test_pong(self):
c = HttpClient()
handler = Echo()
ws = yield c.get(self.ws_echo, websocket_handler=handler).on_headers
#
ws.ping('TESTING CLIENT PING')
message = yield handler.get()
self.assertEqual(message, 'PONG: TESTING CLIENT PING')
def test_verify(self):
c = HttpClient()
yield from self.async.assertRaises(SSLError, c.get, self.httpbin())
response = yield from c.get(self.httpbin(), verify=False)
self.assertEqual(response.status_code, 200)
response = yield from c.get(self.httpbin(), verify=crt)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.request.verify, crt)
def test_ping(self):
c = HttpClient()
handler = Echo()
ws = yield c.get(self.ws_echo, websocket_handler=handler).on_headers
#
# ASK THE SERVER TO SEND A PING FRAME
ws.write('send ping TESTING PING')
message = yield handler.get()
self.assertEqual(message, 'PING: TESTING PING')
def test_ping(self):
c = HttpClient()
handler = Echo()
ws = yield c.get(self.ws_echo, websocket_handler=handler).on_headers
#
# ASK THE SERVER TO SEND A PING FRAME
ws.write('send ping TESTING PING')
message = yield handler.get()
self.assertEqual(message, 'PING: TESTING PING')
def coveralls(http=None, url=None, data_file=None, repo_token=None,
git=None, service_name=None, service_job_id=None,
strip_dirs=None, ignore_errors=False, stream=None):
'''Send a coverage report to coveralls.io.
:param http: optional http client
:param url: optional url to send data to. It defaults to ``coveralls``
api url.
:param data_file: optional data file to load coverage data from. By
default, coverage uses ``.coverage``.
:param repo_token: required when not submitting from travis.
https://coveralls.io/docs/api
'''
stream = stream or sys.stdout
coverage = Coverage(data_file=data_file)
coverage.load()
if http is None:
http = HttpClient(loop=new_event_loop())
if not git:
try:
git = gitrepo()
except Exception: # pragma nocover
pass
data = {'source_files': coverage.coveralls(strip_dirs=strip_dirs,
ignore_errors=ignore_errors)}
if git:
data['git'] = git
if os.environ.get('TRAVIS'):
data['service_name'] = service_name or 'travis-ci'
data['service_job_id'] = os.environ.get('TRAVIS_JOB_ID')
else:
assert repo_token, 'Requires repo_token if not submitting from travis'
if repo_token:
data['repo_token'] = repo_token
url = url or COVERALLS_URL
stream.write('Submitting coverage report to %s\n' % url)
response = http.post(url, files={'json_file': json.dumps(data)})
stream.write('Response code: %s\n' % response.status_code)
try:
info = response.json()
code = 0
if 'error' in info:
stream.write('An error occured while sending coverage'
' report to coverall.io')
code = 1
stream.write('\n%s\n' % info['message'])
except Exception:
code = 1
stream.write('Critical error %s\n' % response.status_code)
return code
def minify(self, options, data):
b = convert_bytes(len(data))
self.write('Minimise %s css file via http://cssminifier.com' % b)
http = HttpClient(loop=new_event_loop())
response = http.post('http://cssminifier.com/raw',
data={'input': data})
if response.status_code == 200:
return native_str(response.get_content())
else:
response.raise_for_status()
def minify(self, options, data):
b = convert_bytes(len(data))
self.write('Minimise %s css file via http://cssminifier.com' % b)
http = HttpClient(loop=new_event_loop())
response = http.post('http://cssminifier.com/raw',
data={'input': data})
if response.status_code == 200:
return native_str(response.get_content())
else:
response.raise_for_status()
def test_dodgy_on_header_event(self):
client = HttpClient()
hook = partial(dodgyhook, self)
response = client.get(self.httpbin(), on_headers=hook)
try:
yield response.on_finished
except ValueError:
pass
self.assertTrue(response.headers)
self.assertIsInstance(response.on_headers.result, Failure)
def test_close(self):
c = HttpClient()
handler = Echo(c._loop)
ws = yield c.get(self.ws_echo, websocket_handler=handler)
self.assertEqual(ws.event('post_request').fired(), 0)
ws.write('send close 1001')
message = yield handler.get()
self.assertEqual(message, 'CLOSE')
self.assertTrue(ws.close_reason)
self.assertEqual(ws.close_reason[0], 1001)
self.assertTrue(ws._connection.closed)
def test_close(self):
c = HttpClient()
handler = Echo()
ws = yield c.get(self.ws_echo, websocket_handler=handler)
self.assertEqual(ws.event('post_request').fired(), 0)
ws.write('send close 1001')
message = yield handler.get()
self.assertEqual(message, 'CLOSE')
self.assertTrue(ws.close_reason)
self.assertEqual(ws.close_reason[0], 1001)
self.assertTrue(ws._connection.closed)
def __init__(self, url, version=None, data=None,
full_response=False, http=None, timeout=None, **kw):
self._url = url
self._version = version or self.__class__.default_version
self._full_response = full_response
self._data = data if data is not None else {}
if not http:
timeout = timeout if timeout is not None else self.default_timeout
http = HttpClient(timeout=timeout, **kw)
http.headers['accept'] = 'application/json, text/*; q=0.5'
http.headers['content-type'] = 'application/json'
self._http = http
def test_home(self):
http = HttpClient()
response = yield from http.get(self.url)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers['content-type'],
'text/html; charset=utf-8')
cookie = response.cookies.get('luxtest')
self.assertTrue(cookie)
self.assertTrue(cookie.value)
response = yield from http.get(self.url)
cookie2 = response.cookies.get('luxtest')
self.assertFalse(cookie2)
def coveralls(http=None,
url=None,
data_file=None,
repo_token=None,
git=None,
service_name=None,
service_job_id=None,
strip_dirs=None,
ignore_errors=False,
stream=None):
'''Send a coverage report to coveralls.io.
:param http: optional http client
:param url: optional url to send data to. It defaults to ``coveralls``
api url.
:param data_file: optional data file to load coverage data from. By
default, coverage uses ``.coverage``.
:param repo_token: required when not submitting from travis.
https://coveralls.io/docs/api
'''
stream = stream or sys.stdout
coverage = Coverage(data_file=data_file)
coverage.load()
if http is None:
from pulsar.apps.http import HttpClient
http = HttpClient(force_sync=True)
if not service_job_id:
service_job_id = os.environ.get('TRAVIS_JOB_ID', '')
if service_job_id:
service_name = 'travis-ci'
if not git:
git = gitrepo()
data = {
'service_job_id': service_job_id,
'service_name': service_name or 'pulsar',
'git': git,
'source_files': coverage.coveralls(strip_dirs, ignore_errors),
}
if repo_token:
data['repo_token'] = repo_token
url = url or COVERALLS_URL
stream.write('Submitting coverage report to %s\n' % url)
response = http.post(url, files={'json_file': json.dumps(data)})
stream.write('Response code: %s\n' % response.status_code)
info = response.json()
if 'error' in info:
stream.write('An error occured while sending coverage'
' report to coverall.io')
stream.write('\n%s' % info['message'])
def test_close_sync(self):
loop = new_event_loop()
c = HttpClient(loop=loop)
handler = Echo(loop)
ws = c.get(self.ws_echo, websocket_handler=handler)
self.assertEqual(ws.event('post_request').fired(), 0)
self.assertEqual(ws._loop, loop)
self.assertFalse(ws._loop.is_running())
ws.write('send close 1001')
message = ws._loop.run_until_complete(handler.get())
self.assertEqual(message, 'CLOSE')
self.assertTrue(ws.close_reason)
self.assertEqual(ws.close_reason[0], 1001)
self.assertTrue(ws._connection.closed)
def test_close_sync(self):
loop = new_event_loop()
c = HttpClient(loop=loop)
handler = Echo(loop)
ws = c.get(self.ws_echo, websocket_handler=handler)
self.assertEqual(ws.event('post_request').fired(), 0)
self.assertEqual(ws._loop, loop)
self.assertFalse(ws._loop.is_running())
ws.write('send close 1001')
message = ws._loop.run_until_complete(handler.get())
self.assertEqual(message, 'CLOSE')
self.assertTrue(ws.close_reason)
self.assertEqual(ws.close_reason[0], 1001)
self.assertTrue(ws._connection.closed)
def monitor_start(self, monitor):
# this application has no workers
self.cfg.set('workers', 0)
api_key = self.get_param('twitter_api_key')
client_secret = self.get_param('twitter_api_secret')
access_token = self.get_param('twitter_access_token')
access_secret = self.get_param('twitter_access_secret')
self._http = HttpClient(encode_multipart=False)
oauth2 = OAuth1(api_key,
client_secret=client_secret,
resource_owner_key=access_token,
resource_owner_secret=access_secret)
self._http.bind_event('pre_request', oauth2)
self.buffer = []
self.connect()
def worker_start(self, worker, exc=None):
'''When the worker starts :meth:`connect` to twitter spreaming
endpoint
'''
api_key = self.get_param('twitter_api_key')
client_secret = self.get_param('twitter_api_secret')
access_token = self.get_param('twitter_access_token')
access_secret = self.get_param('twitter_access_secret')
self._http = HttpClient(encode_multipart=False)
oauth1 = OAuth1(api_key,
client_secret=client_secret,
resource_owner_key=access_token,
resource_owner_secret=access_secret)
self._http.bind_event('pre_request', oauth1)
self.buffer = []
ensure_future(self.connect())
def test_override_headers(self):
headers = {'Accept': 'application/json, text/plain; q=0.8',
'content-type': 'application/json'}
client = HttpClient(headers=headers)
self.assertEqual(client.headers['accept'],
'application/json, text/plain; q=0.8')
self.assertEqual(client.headers['content-type'], 'application/json')
def testBadRequests(self):
c = HttpClient()
response = yield c.post(self.ws_uri)
self.assertEqual(response.status_code, 405)
#
response = yield c.get(self.ws_uri,
headers=[('Sec-Websocket-Key', 'x')])
self.assertEqual(response.status_code, 400)
#
response = yield c.get(self.ws_uri,
headers=[('Sec-Websocket-Key', 'bla')])
self.assertEqual(response.status_code, 400)
#
response = yield c.get(self.ws_uri,
headers=[('Sec-Websocket-version', 'xxx')])
self.assertEqual(response.status_code, 400)
def setUpClass(cls):
s = server(name=cls.name(),
concurrency=cls.concurrency,
bind='127.0.0.1:0')
cls.app_cfg = yield from send('arbiter', 'run', s)
cls.uri = 'http://{0}:{1}'.format(*cls.app_cfg.addresses[0])
cls.client = HttpClient()
def http(self):
if self._parent:
return self._parent.http
else:
if self._http is None:
self._http = HttpClient(loop=asyncio.new_event_loop())
return self._http
def test_upgrade(self):
c = HttpClient()
handler = Echo()
ws = yield c.get(self.ws_echo, websocket_handler=handler).on_headers
response = ws.handshake
self.assertEqual(response.status_code, 101)
self.assertEqual(response.headers['upgrade'], 'websocket')
self.assertEqual(ws.connection, response.connection)
self.assertEqual(ws.handler, handler)
#
# on_finished
self.assertFalse(response.on_finished.done())
self.assertFalse(ws.on_finished.done())
# Send a message to the websocket
ws.write('Hi there!')
message = yield handler.get()
self.assertEqual(message, 'Hi there!')
def __init__(self, http=None, **kwargs):
if http is None:
http = HttpClient()
self.http = http
if self.json:
self.http.headers['content-type'] = 'application/json'
self.http.headers['accept'] = 'application/json'
self.setup(**kwargs)
def test_urlparams(self):
http = HttpClient()
params = {'page': 2, 'key': 'foo'}
request = HttpRequest(http, 'http://bla.com?k=6', 'post',
params=params)
url = urlparse(request.url)
data = parse_qsl(url.query)
self.assertEqual(len(data), 3)
async def test_oauth1(self):
oauth = OAuth1('random',
client_secret='xxxxxxx',
resource_owner_key='xxxxxxx',
resource_owner_secret='xxxxxxx')
async with HttpClient() as http:
await http.post('https://api.github.com/gists/public',
pre_request=oauth)
async def setUpClass(cls):
s = FlaskGreen(name=cls.name(),
concurrency=cls.concurrency,
bind='127.0.0.1:0',
echo_bind='127.0.0.1:0')
cls.app_cfg = await send('arbiter', 'run', s)
cls.uri = 'http://{0}:{1}'.format(*cls.app_cfg[0].addresses[0])
cls.client = HttpClient()
async def test_verify(self):
c = HttpClient()
await self.wait.assertRaises(SSLError, c.get, self.httpbin())
response = await c.get(self.httpbin(), verify=False)
self.assertEqual(response.status_code, 200)
response = await c.get(self.httpbin(), verify=crt)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.request.verify, crt)
def test_upgrade(self):
c = HttpClient()
handler = Echo()
ws = yield c.get(self.ws_echo, websocket_handler=handler).on_headers
response = ws.handshake
self.assertEqual(response.status_code, 101)
self.assertEqual(response.headers['upgrade'], 'websocket')
self.assertEqual(ws.connection, response.connection)
self.assertEqual(ws.handler, handler)
#
# on_finished
self.assertFalse(response.on_finished.done())
self.assertFalse(ws.on_finished.done())
# Send a message to the websocket
ws.write('Hi there!')
message = yield handler.get()
self.assertEqual(message, 'Hi there!')
def __init__(self,
url,
version=None,
data=None,
full_response=False,
http=None,
timeout=None,
**kw):
self._url = url
self._version = version or self.__class__.default_version
self._full_response = full_response
self._data = data if data is not None else {}
if not http:
timeout = timeout if timeout is not None else self.default_timeout
http = HttpClient(timeout=timeout, **kw)
http.headers['accept'] = 'application/json, text/*; q=0.5'
http.headers['content-type'] = 'application/json'
self._http = http
def client(cls, loop=None, parser=None, pool_size=2, **kwargs):
parser = cls.parser()
if cls.with_proxy:
kwargs['proxy_info'] = {'http': cls.proxy_uri,
'https': cls.proxy_uri,
'ws': cls.proxy_uri,
'wss': cls.proxy_uri}
return HttpClient(loop=loop, parser=parser, pool_size=pool_size,
**kwargs)
def http_sessions(self, model=None):
"""Return an HTTP session handler for a given concurrency model
"""
if model == concurrency.THREAD_IO:
return HttpClient(loop=new_event_loop())
elif model == concurrency.ASYNC_IO:
return self.http
else:
return GreenHttp(self.http)
async def create(cls, cfg, loop=None, auth=None):
http = HttpClient(loop=loop)
try:
git = await Git.create()
gitapi = git.api(auth=auth, http=http)
repo_path = await git.toplevel()
except Exception:
git = gitapi = repo_path = None
return cls(cfg, http, git, gitapi, repo_path)
def media_libraries():
global _media_libraries
if _media_libraries is None:
if os.path.isfile('libs.json'): # pragma nocover
with open('libs.json') as f:
data = f.read()
_media_libraries = json.loads(data)
else:
from pulsar import new_event_loop
from pulsar.apps.http import HttpClient
http = HttpClient(loop=new_event_loop())
try:
response = http.get(_libs_url)
_media_libraries = response.json()
except Exception: # pragma nocover
http.logger.error('Could not import media library',
exc_info=True)
_media_libraries = {'libs': {}, 'deps': {}}
return _media_libraries
def __init__(self, auth=None, http=None):
if not http:
http = HttpClient()
self.logger = logging.getLogger('agile.github')
self.http = http
try:
self.auth = auth or get_auth()
except utils.AgileError as exc:
self.logger.warning(str(exc))
self.auth = None
def media_libraries():
global _media_libraries
if _media_libraries is None:
if os.path.isfile('libs.json'): # pragma nocover
with open('libs.json') as f:
data = f.read()
_media_libraries = json.loads(data)
else:
from pulsar import new_event_loop
from pulsar.apps.http import HttpClient
http = HttpClient(loop=new_event_loop())
try:
response = http.get(_libs_url)
_media_libraries = response.json()
except Exception: # pragma nocover
http.logger.error('Could not import media library',
exc_info=True)
_media_libraries = {'libs': {}, 'deps': {}}
return _media_libraries
def coveralls(http=None, url=None, data_file=None, repo_token=None, git=None,
service_name=None, service_job_id=None, strip_dirs=None,
ignore_errors=False, stream=None):
'''Send a coverage report to coveralls.io.
:param http: optional http client
:param url: optional url to send data to. It defaults to ``coveralls``
api url.
:param data_file: optional data file to load coverage data from. By
default, coverage uses ``.coverage``.
:param repo_token: required when not submitting from travis.
https://coveralls.io/docs/api
'''
stream = stream or sys.stdout
coverage = Coverage(data_file=data_file)
coverage.load()
if http is None:
from pulsar.apps.http import HttpClient
http = HttpClient(force_sync=True)
if not service_job_id:
service_job_id = os.environ.get('TRAVIS_JOB_ID', '')
if service_job_id:
service_name = 'travis-ci'
if not git:
git = gitrepo()
data = {
'service_job_id': service_job_id,
'service_name': service_name or 'pulsar',
'git': git,
'source_files': coverage.coveralls(strip_dirs, ignore_errors),
}
if repo_token:
data['repo_token'] = repo_token
url = url or COVERALLS_URL
stream.write('Submitting coverage report to %s\n' % url)
response = http.post(url, files={'json_file': json.dumps(data)})
stream.write('Response code: %s\n' % response.status_code)
info = response.json()
if 'error' in info:
stream.write('An error occured while sending coverage'
' report to coverall.io')
stream.write('\n%s' % info['message'])
def start(self, worker):
self.http = HttpClient(loop=self._loop)
self.accounts = {}
api_key = self.get_param('twitter_api_key')
api_secret = self.get_param('twitter_api_secret')
twitter_accounts = self.get_param('twitter_accounts')
for account in twitter_accounts:
account = Account(self, api_key, api_secret, account)
self.accounts[account.token] = account
account.connect()
def http(self):
'''Get the HTTP client
'''
if self._http is None:
api_url = self.app.config['API_URL']
headers = [('content-type', 'application/json')]
# Remote API
if is_absolute_uri(api_url):
if self.app.green_pool:
self._http = HttpClient(headers=headers)
else:
self._http = HttpClient(loop=new_event_loop())
# Local API
else:
self._http = LocalClient(self.app, headers)
token = self.app.config['API_AUTHENTICATION_TOKEN']
if token:
self._http.headers['Athentication'] = 'Bearer %s' % token
return self._http
def _init(self, headers=None, doc_url=None, **kw):
bits = self._name.split('+')
if len(bits) == 2:
self._scheme = bits[0]
self._name = bits[1]
else:
self._scheme = 'http'
host = self._host
if isinstance(host, tuple):
host = '%s:%s' % host
self._address = '%s://%s' % (self._scheme, host)
self._http = HttpClient(loop=self._loop, **kw)
def client(cls,
loop=None,
parser=None,
pool_size=2,
verify=False,
**kwargs):
parser = cls.parser()
kwargs['proxies'] = cls.proxies()
return HttpClient(loop=loop,
parser=parser,
pool_size=pool_size,
verify=verify,
**kwargs)
def monitor_start(self, monitor):
# this application has no workers
self.cfg.set('workers', 0)
api_key = self.get_param('twitter_api_key')
client_secret = self.get_param('twitter_api_secret')
access_token = self.get_param('twitter_access_token')
access_secret = self.get_param('twitter_access_secret')
self._http = HttpClient(encode_multipart=False)
oauth2 = OAuth1(api_key,
client_secret=client_secret,
resource_owner_key=access_token,
resource_owner_secret=access_secret)
self._http.bind_event('pre_request', oauth2)
self.buffer = []
self.connect()
def worker_start(self, worker, exc=None):
'''When the worker starts :meth:`connect` to twitter spreaming
endpoint
'''
api_key = self.get_param('twitter_api_key')
client_secret = self.get_param('twitter_api_secret')
access_token = self.get_param('twitter_access_token')
access_secret = self.get_param('twitter_access_secret')
self._http = HttpClient(encode_multipart=False)
oauth1 = OAuth1(api_key,
client_secret=client_secret,
resource_owner_key=access_token,
resource_owner_secret=access_secret)
self._http.bind_event('pre_request', oauth1)
self.buffer = []
self.connect()
def test_home(self):
c = HttpClient()
response = yield c.get(self.uri)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers['content-type'],
'text/html; charset=utf-8')
class Twitter(pulsar.Application):
'''This applications requires the following parameters
to be specified in your ``config.py`` file:
* ``twitter_api_key`` the Consumer key of your application
* ``twitter_api_secret``, the Consumer secret
* ``twitter_access_token``, the application Access token
* ``twitter_access_secret``, the Access token secret
* ``twitter_stream_filter``, dictionary of parameters for
filtering tweets.
Check the twitter stream filter api for further information
https://dev.twitter.com/docs/api/1.1/post/statuses/filter
'''
interval1 = 0
interval2 = 0
interval3 = 0
public_stream = 'https://stream.twitter.com/1.1/statuses/filter.json'
def monitor_start(self, monitor):
# this application has no workers
self.cfg.set('workers', 0)
api_key = self.get_param('twitter_api_key')
client_secret = self.get_param('twitter_api_secret')
access_token = self.get_param('twitter_access_token')
access_secret = self.get_param('twitter_access_secret')
self._http = HttpClient(encode_multipart=False)
oauth2 = OAuth1(api_key,
client_secret=client_secret,
resource_owner_key=access_token,
resource_owner_secret=access_secret)
self._http.bind_event('pre_request', oauth2)
self.buffer = []
self.connect()
def connect(self):
filter = self.get_param('twitter_stream_filter')
self._http.post(self.public_stream, data=filter,
on_headers=self.connected,
data_processed=self.process_data,
post_request=self.reconnect)
def connected(self, response, **kw):
if response.status_code == 200:
self.logger.info('Successfully connected with twitter streaming')
self.interval1 = 0
self.interval2 = 0
self.interval3 = 0
def process_data(self, response, **kw):
if response.status_code == 200:
messages = []
data = response.recv_body()
while data:
idx = data.find(b'\r\n')
if idx < 0:
self.buffer.append(data)
data = None
else:
self.buffer.append(data[:idx])
data = data[idx+2:]
msg = b''.join(self.buffer)
self.buffer = []
if msg:
body = json.loads(msg.decode('utf-8'))
if 'disconnect' in body:
msg = body['disconnect']
self.logger.warning('Disconnecting (%d): %s',
msg['code'], msg['reason'])
elif 'warning' in body:
message = body['warning']['message']
self.logger.warning(message)
else:
messages.append(body)
if messages:
# a list of messages is available
if self.cfg.callable:
self.cfg.callable(messages)
def reconnect(self, response, exc=None):
'''Handle reconnection according to twitter streaming policy
https://dev.twitter.com/docs/streaming-apis/connecting
'''
loop = response._loop
if response.status_code == 200:
gap = 0
elif not response.status_code:
# This is a network error, back off lineraly 250ms up to 16s
self.interval1 = gap = max(self.interval1+0.25, 16)
elif response.status_code == 420:
gap = 60 if not self.interval2 else max(2*self.interval2)
self.interval2 = gap
else:
gap = 5 if not self.interval3 else max(2*self.interval3, 320)
self.interval3 = gap
loop.call_later(gap, self.connect)
def get_param(self, name):
value = self.cfg.get(name)
if not value:
raise pulsar.ImproperlyConfigured(
'Please specify the "%s" parameter in your %s file' %
(name, self.cfg.config))
return value
def test_reset_password(self):
http = HttpClient()
url = self.url + self.app.config['RESET_PASSWORD_URL']
response = yield from http.get(url)
self.assertEqual(response.status_code, 200)
def test_404(self):
http = HttpClient()
url = self.url + '/dkvshcvsdkchsdkc'
response = yield from http.get(url)
self.assertEqual(response.status_code, 404)
class Twitter(pulsar.Application):
'''A pulsar :class:`.Application` for connecting and streaming
tweets from `twitter streaming api`_.
This application requires the following parameters
to be specified in your ``config.py`` file:
* ``twitter_api_key`` the Consumer key of your application
* ``twitter_api_secret``, the Consumer secret
* ``twitter_access_token``, the application Access token
* ``twitter_access_secret``, the Access token secret
* ``twitter_stream_filter``, dictionary of parameters for
`filtering tweets`_.
'''
interval1 = 0
interval2 = 0
interval3 = 0
public_stream = 'https://stream.twitter.com/1.1/statuses/filter.json'
def monitor_start(self, monitor):
'''When the monitor starts make sure only one worker at most
is spawned
'''
self.cfg.set('workers', min(self.cfg.workers, 1))
def worker_start(self, worker, exc=None):
'''When the worker starts :meth:`connect` to twitter spreaming
endpoint
'''
api_key = self.get_param('twitter_api_key')
client_secret = self.get_param('twitter_api_secret')
access_token = self.get_param('twitter_access_token')
access_secret = self.get_param('twitter_access_secret')
self._http = HttpClient(encode_multipart=False)
oauth1 = OAuth1(api_key,
client_secret=client_secret,
resource_owner_key=access_token,
resource_owner_secret=access_secret)
self._http.bind_event('pre_request', oauth1)
self.buffer = []
self.connect()
@task
def connect(self):
'''Connect to twitter streaming endpoint.
If the connection is dropped, the :meth:`reconnect` method is invoked
according to twitter streaming connection policy_.
'''
filter = self.get_param('twitter_stream_filter')
return self._http.post(self.public_stream,
data=filter,
on_headers=self.connected,
data_processed=self.process_data,
post_request=self.reconnect)
def connected(self, response, **kw):
'''Callback when a succesful connection is made.
Reset reconnection intervals to 0
'''
if response.status_code == 200:
self.logger.info('Successfully connected with twitter streaming')
self.interval1 = 0
self.interval2 = 0
self.interval3 = 0
def process_data(self, response, **kw):
'''Callback passed to :class:`HttpClient` for processing
streaming data.
'''
if response.status_code == 200:
messages = []
data = response.recv_body()
while data:
idx = data.find(b'\r\n')
if idx < 0: # incomplete data - add to buffer
self.buffer.append(data)
data = None
else:
self.buffer.append(data[:idx])
data = data[idx+2:]
msg = b''.join(self.buffer)
self.buffer = []
if msg:
body = json.loads(msg.decode('utf-8'))
if 'disconnect' in body:
msg = body['disconnect']
self.logger.warning('Disconnecting (%d): %s',
msg['code'], msg['reason'])
elif 'warning' in body:
message = body['warning']['message']
self.logger.warning(message)
else:
messages.append(body)
if messages:
# a list of messages is available
if self.cfg.callable:
self.cfg.callable(self, messages)
def reconnect(self, response, exc=None):
'''Handle reconnection according to twitter streaming policy_
.. _policy: https://dev.twitter.com/docs/streaming-apis/connecting
'''
loop = response._loop
if response.status_code == 200:
gap = 0
elif not response.status_code:
# This is a network error, back off lineraly 250ms up to 16s
self.interval1 = gap = max(self.interval1+0.25, 16)
elif response.status_code == 420:
gap = 60 if not self.interval2 else max(2*self.interval2)
self.interval2 = gap
else:
if response.status_code >= 400:
self.logger.error('Could not connect to twitter spreaming API,'
' status code %s' % response.status_code)
gap = 5 if not self.interval3 else max(2*self.interval3, 320)
self.interval3 = gap
loop.call_later(gap, self.connect)
def get_param(self, name):
value = self.cfg.get(name)
if not value:
raise pulsar.ImproperlyConfigured(
'Please specify the "%s" parameter in your %s file' %
(name, self.cfg.config))
return value
