def set_config(local_config, ui_config, callback):
scheme = local_config.get('@scheme', local_config.get('scheme'))
host = local_config.get('@host', local_config.get('host'))
port = local_config.get('@port', local_config.get('port'))
username = local_config.get('@username', local_config.get('username'))
password = local_config.get('@password', local_config.get('password'))
path = local_config.get('@path', local_config.get('path')) or ''
camera_id = local_config.get('@remote_camera_id', local_config.get('remote_camera_id'))
logging.debug('setting config for remote camera %(id)s on %(url)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config)})
ui_config = json.dumps(ui_config)
request = _make_request(scheme, host, port, username, password,
path + '/config/%(id)s/set/' % {'id': camera_id},
method='POST', data=ui_config, content_type='application/json')
def on_response(response):
if response.error:
logging.error('failed to set config for remote camera %(id)s on %(url)s: %(msg)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
callback()
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def set_preview(local_config, controls, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('setting preview for remote camera %(id)s on %(url)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config)})
data = json.dumps(controls)
request = _make_request(scheme, host, port, username, password,
path + '/config/%(id)s/set_preview/' % {'id': camera_id},
method='POST', data=data, content_type='application/json')
def on_response(response):
if response.error:
logging.error('failed to set preview for remote camera %(id)s on %(url)s: %(msg)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
callback()
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def async_http(self, proxy, idx, requests_proxies):
success_count = self.SUCCESS_RATIO * self.CONNECTIVITY_TEST
fail_count = self.CONNECTIVITY_TEST - success_count
count = {
} # callback of async_httpclient(handle_request) get local variable,
# no conflict with multi coroutine, originally
count[idx] = [success_count, fail_count]
for protocol, pro_ip_port in proxy.items():
pro_type, host, port = pro_ip_port
if protocol == pycurl.PROXYTYPE_HTTP:
pass
elif protocol == pycurl.PROXYTYPE_SOCKS5:
pass
elif protocol == pycurl.PROXYTYPE_SOCKS4:
pass
else:
yield
def prepare_curl_type(curl):
curl.setopt(pycurl.PROXYTYPE, protocol)
@gen.coroutine
def handle_request(response):
if response.code == 200:
count[idx][0] -= 1
else:
count[idx][1] -= 1
AsyncHTTPClient.configure(
'tornado.curl_httpclient.CurlAsyncHTTPClient')
http_client = AsyncHTTPClient()
http_request = httpclient.HTTPRequest(
url=self.TESTURL,
method='HEAD',
proxy_host=host,
proxy_port=int(port),
prepare_curl_callback=prepare_curl_type,
follow_redirects=False,
)
for i in range(self.CONNECTIVITY_TEST):
try:
response = yield http_client.fetch(http_request,
handle_request)
except CurlError as e:
print('Curl Error: ', e)
while True:
gen.sleep(0.01)
if count[idx][0] <= 0:
if pro_type == 'https':
requests_proxies.add('{}://{}:{}'.format(
pro_type, host, port))
else:
requests_proxies.add('{}://{}:{}'.format(
pro_type, host, port))
raise gen.Return(True)
if count[idx][1] <= 0:
raise gen.Return(False)
class Scour(object):
"The base class of the scraper"
def __init__(self, seed_urls=[], url_fetch_timeout=25):
ioloop.IOLoop.instance() #instanceiate IOLoop
self.client = CurlAsyncHTTPClient()
logging.basicConfig(level=logging.DEBUG, filename="m_scrape.log", filemode='w')
self.log = logging.getLogger(name="mod_scrape")
self.__extractors = []
#Queue Argumet urls
for url in seed_urls:
self.get(url)
def get(self, url):
self.client.fetch(str(url), self.process_page)
def process_page(self, response):
print "processing page"
try:
soup = bSoup(response.body)
for x in self.__extractors: #Run the extractors
x(self, soup, response)
except Exception, e: #Log any errors
import traceback
self.log.error(str(traceback.format_exc(limit=10)))
self.log.error(str(e))
def del_media_group(local_config, group, media_type, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('deleting group "%(group)s" of remote camera %(id)s on %(url)s' % {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config)})
path += '/%(media_type)s/%(id)s/delete_all/%(group)s/' % {
'media_type': media_type,
'id': camera_id,
'group': group}
request = _make_request(scheme, host, port, username, password, path,
method='POST', data='{}',
timeout=settings.REMOTE_REQUEST_TIMEOUT, content_type='application/json')
def on_response(response):
if response.error:
logging.error('failed to delete group "%(group)s" of remote camera %(id)s on %(url)s: %(msg)s' % {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
callback()
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def check_timelapse_movie(local_config, group, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('checking timelapse movie status for remote camera %(id)s on %(url)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config)})
request = _make_request(scheme, host, port, username, password,
path + '/picture/%(id)s/timelapse/%(group)s/?check=true' % {
'id': camera_id,
'group': group})
def on_response(response):
if response.error:
logging.error('failed to check timelapse movie status for remote camera %(id)s on %(url)s: %(msg)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)})
return callback(error=unicode(e))
callback(response)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
class jarvis_testcase(tornado.testing.AsyncHTTPTestCase):
def setUp(self):
super(jarvis_testcase, self).setUp()
self.http_client = CurlAsyncHTTPClient(self.io_loop)
self.headers = {'secret': config.config['secret']}
def get_app(self):
self.jarvis = kernel.init(config.config)
return self.jarvis._application
@tornado.testing.gen_test
def http_request(self, request, headers = None):
command = urllib.quote(request, '')
uri = command.replace(urllib.quote(' '), '/', 2)
url = '/api/' + uri
if headers == None:
headers = self.headers
request = HTTPRequest(self.get_url(url), headers=headers)
response = yield self.http_client.fetch(request, raise_error=False)
raise tornado.gen.Return(json.loads(response.body))
@tornado.testing.gen_test
def raw_http_request(self, url, headers = None):
if headers == None:
headers = self.headers
request = HTTPRequest(self.get_url(url), headers=headers)
response = yield self.http_client.fetch(request, raise_error=False)
raise tornado.gen.Return(response)
def exec_action(local_config, action, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('executing action "%(action)s" of remote camera %(id)s on %(url)s' % {
'action': action,
'id': camera_id,
'url': pretty_camera_url(local_config)})
path += '/action/%(id)s/%(action)s/' % {
'action': action,
'id': camera_id}
request = _make_request(scheme, host, port, username, password, path,
method='POST', data='{}',
timeout=settings.REMOTE_REQUEST_TIMEOUT, content_type='application/json')
def on_response(response):
if response.error:
logging.error('failed to execute action "%(action)s" of remote camera %(id)s on %(url)s: %(msg)s' % {
'action': action,
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
callback()
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
class CurlHTTPClientTestCase(AsyncHTTPTestCase):
def setUp(self):
super(CurlHTTPClientTestCase, self).setUp()
self.http_client = CurlAsyncHTTPClient(self.io_loop)
def get_app(self):
return Application([
('/digest', DigestAuthHandler),
])
def test_prepare_curl_callback_stack_context(self):
exc_info = []
def error_handler(typ, value, tb):
exc_info.append((typ, value, tb))
self.stop()
return True
with ExceptionStackContext(error_handler):
request = HTTPRequest(self.get_url('/'),
prepare_curl_callback=lambda curl: 1 / 0)
self.http_client.fetch(request, callback=self.stop)
self.wait()
self.assertEqual(1, len(exc_info))
self.assertIs(exc_info[0][0], ZeroDivisionError)
def test_digest_auth(self):
response = self.fetch('/digest',
auth_mode='digest',
auth_username='******',
auth_password='******')
self.assertEqual(response.body, b'ok')
def notice_lotto_result(lotto_id, issue, numbers):
post_data = {
"lotto_id": lotto_id,
"issue": issue,
"numbers": numbers,
}
address_info_list = lru_db_find_lotto_push({"status": 1})
def handle_response(response):
if response.error:
# print response.code
warn(get_string(response.error))
pass
else:
try:
result = loads(response.body)
code = get_int(result.get("code"), -1)
msg = get_string(result.get("msg"))
if code == 0:
info("==>%s", msg)
else:
warn("=>%s", msg)
except Exception as e:
import traceback
print traceback.format_exc()
print e
for address_info in address_info_list:
curl_client = CurlAsyncHTTPClient()
url = address_info.get("address")
request = HTTPRequest(url=url,
method='POST',
body=urllib.urlencode(post_data))
curl_client.fetch(request, handle_response)
def fetch(self, url, **kwargs):
# init HTTPRequest
http_client = CurlAsyncHTTPClient()
session = HTTPRequest('', follow_redirects=False)
instance_parameters = copy.deepcopy(self._req_params) # 参数
self.init_request(session, url, **kwargs)
while True:
self.pre_request(session, url=url, **kwargs)
try:
fetch_logger.info('{method} {url}'.format(method=session.method, url=session.url))
response = yield http_client.fetch(session)
fetch_logger.log_green('{code} {url}'.format(code=response.code, url=session.url))
break
except HTTPError as httperr:
# redirects handler
if httperr.code > 300 and httperr.code < 400:
fetch_logger.warning('{code} {url}'.format(code=httperr.code, url=session.url))
self.post_request(session, httperr.response, url, **kwargs)
if not kwargs.get('disabled_redirect'):
url = httperr.response.headers.get('Location')
else:
fetch_logger.debug(httperr)
raise gen.Return(httperr.response)
else:
fetch_logger.error(httperr)
raise httperr
del instance_parameters
self.post_request(session, response, url, **kwargs)
raise gen.Return(response)
def get_html(url, params=None):
if params is None:
params = {}
future = Future()
def handle_response(response):
if response.error:
# future.set_exception(response.error)
future.set_result('')
else:
future.set_result(response.body)
curl_client = CurlAsyncHTTPClient()
headers = {
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
"Accept-Language":
"zh-CN,zh;q=0.8,en;q=0.6",
"Cache-Control":
"max-age=0",
}
if params.get('headers'):
headers = params.get('headers')
request = HTTPRequest(url=url, headers=headers)
if params.get("host") and params.get("port"):
request.proxy_host = params.get("host")
request.proxy_port = get_int(params.get("port"))
curl_client.fetch(request, handle_response)
return future
def get(self):
request_client = HTTPRequest(url = self.request.uri,
method = self.request.method,
body = self.request.body or None,
headers = self.request.headers)
http_client = CurlAsyncHTTPClient()
response = http_client.fetch(request_client, self.response_client)
def test(local_config, data, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
what = data['what']
logging.debug('testing %(what)s on remote camera %(id)s, on %(url)s' % {
'what': what,
'id': camera_id,
'url': pretty_camera_url(local_config)})
data = json.dumps(data)
request = _make_request(scheme, host, port, username, password,
path + '/config/%(id)s/test/' % {'id': camera_id},
method='POST', data=data, content_type='application/json')
def on_response(response):
if response.error:
logging.error('failed to test %(what)s on remote camera %(id)s, on %(url)s: %(msg)s' % {
'what': what,
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
callback()
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
class Scour(object):
"The base class of the scraper"
def __init__(self, seed_urls=[], url_fetch_timeout=25):
ioloop.IOLoop.instance() #instanceiate IOLoop
self.client = CurlAsyncHTTPClient()
logging.basicConfig(level=logging.DEBUG,
filename="m_scrape.log",
filemode='w')
self.log = logging.getLogger(name="mod_scrape")
self.__extractors = []
#Queue Argumet urls
for url in seed_urls:
self.get(url)
def get(self, url):
self.client.fetch(str(url), self.process_page)
def process_page(self, response):
print "processing page"
try:
soup = bSoup(response.body)
for x in self.__extractors: #Run the extractors
x(self, soup, response)
except Exception, e: #Log any errors
import traceback
self.log.error(str(traceback.format_exc(limit=10)))
self.log.error(str(e))
class jarvis_testcase(tornado.testing.AsyncHTTPTestCase):
def setUp(self):
super(jarvis_testcase, self).setUp()
self.http_client = CurlAsyncHTTPClient(self.io_loop)
self.headers = {'secret': config.config['secret']}
def get_app(self):
self.jarvis = kernel.init(config.config)
return self.jarvis._application
@tornado.testing.gen_test
def http_request(self, request, headers=None):
command = urllib.quote(request, '')
uri = command.replace(urllib.quote(' '), '/', 2)
url = '/api/' + uri
if headers == None:
headers = self.headers
request = HTTPRequest(self.get_url(url), headers=headers)
response = yield self.http_client.fetch(request, raise_error=False)
raise tornado.gen.Return(json.loads(response.body))
@tornado.testing.gen_test
def raw_http_request(self, url, headers=None):
if headers == None:
headers = self.headers
request = HTTPRequest(self.get_url(url), headers=headers)
response = yield self.http_client.fetch(request, raise_error=False)
raise tornado.gen.Return(response)
def make_timelapse_movie(local_config, framerate, interval, group, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(
local_config)
logging.debug(
'making timelapse movie for group "%(group)s" of remote camera %(id)s with rate %(framerate)s/%(int)s on %(url)s'
% {
'group': group or 'ungrouped',
'id': camera_id,
'framerate': framerate,
'int': interval,
'url': pretty_camera_url(local_config)
})
path += '/picture/%(id)s/timelapse/%(group)s/?interval=%(int)s&framerate=%(framerate)s' % {
'id': camera_id,
'int': interval,
'framerate': framerate,
'group': group
}
request = _make_request(scheme,
host,
port,
username,
password,
path,
timeout=100 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error(
'failed to make timelapse movie for group "%(group)s" of remote camera %(id)s with rate %(framerate)s/%(int)s on %(url)s: %(msg)s'
% {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config),
'int': interval,
'framerate': framerate,
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error(
'failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)
})
return callback(error=unicode(e))
callback(response)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def get_media_content(local_config, filename, media_type, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('downloading file %(filename)s of remote camera %(id)s on %(url)s' % {
'filename': filename,
'id': camera_id,
'url': pretty_camera_url(local_config)})
path += '/%(media_type)s/%(id)s/download/%(filename)s' % {
'media_type': media_type,
'id': camera_id,
'filename': filename}
# timeout here is 10 times larger than usual - we expect a big delay when fetching the media list
request = _make_request(scheme, host, port, username, password,
path, timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error('failed to download file %(filename)s of remote camera %(id)s on %(url)s: %(msg)s' % {
'filename': filename,
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
return callback(response.body)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def get_real_ip(self, geolocate=True):
try:
self.real_ip = self.request.headers.get(
'X-Real-Ip',
self.request.headers.get('X-Forwarded-For', None))
logging.info(
"Request from " + str(self.real_ip) + str(self.__class__))
if geolocate:
geo_key = "geo_%s" % self.real_ip
cached_geo = memcache_get(geo_key)
if cached_geo:
logging.info(cached_geo)
else:
def handle_request(responses):
geo = load_json(responses.body)
memcache_set(geo_key, geo)
logging.info(geo)
http_client = CurlAsyncHTTPClient()
# need to make this a external configuration
http_client.fetch("http://freegeoip.net/json/%s" %
self.real_ip,
callback=handle_request,
request_timeout=2,
connect_timeout=2)
except Exception as e:
self.error(e)
def get_timelapse_movie(local_config, key, group, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('downloading timelapse movie for remote camera %(id)s on %(url)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config)})
request = _make_request(scheme, host, port, username, password,
path + '/picture/%(id)s/timelapse/%(group)s/?key=%(key)s' % {
'id': camera_id,
'group': group,
'key': key},
timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error('failed to download timelapse movie for remote camera %(id)s on %(url)s: %(msg)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
callback({
'data': response.body,
'content_type': response.headers.get('Content-Type'),
'content_disposition': response.headers.get('Content-Disposition')
})
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
class CurlHTTPClientTestCase(AsyncHTTPTestCase):
def setUp(self):
super(CurlHTTPClientTestCase, self).setUp()
self.http_client = CurlAsyncHTTPClient(self.io_loop)
def get_app(self):
return Application([
('/digest', DigestAuthHandler),
])
def test_prepare_curl_callback_stack_context(self):
exc_info = []
def error_handler(typ, value, tb):
exc_info.append((typ, value, tb))
self.stop()
return True
with ExceptionStackContext(error_handler):
request = HTTPRequest(self.get_url('/'),
prepare_curl_callback=lambda curl: 1 / 0)
self.http_client.fetch(request, callback=self.stop)
self.wait()
self.assertEqual(1, len(exc_info))
self.assertIs(exc_info[0][0], ZeroDivisionError)
def test_digest_auth(self):
response = self.fetch('/digest', auth_mode='digest',
auth_username='******', auth_password='******')
self.assertEqual(response.body, b'ok')
def __init__(self,dev_callback):
self.url_list = Default_URL;
self.http_client = CurlAsyncHTTPClient()
self.stdudents_update_buf = []
self.stdudents_dele_buf = []
self.db_handler = DB_Handler
self.settings = self.db_handler.get_serverConnection_setting(Default_ServerConnection_Setting)
self.dev_process =dev_callback
def __init__(self, max_clients=100, renderer=None, renderer_cores=None):
self._max_clients = max_clients
self._http_client = CurlAsyncHTTPClient(max_clients=max_clients,
force_instance=True)
self._renderer = renderer
if renderer_cores is None:
renderer_cores = self._max_clients
self._renderer_semaphore = Semaphore(renderer_cores)
def get(self):
request_client = HTTPRequest(url = self.request.uri,
method = self.request.method,
body = self.request.body or None,
headers = self.request.headers,
proxy_host = options.host_proxy,
proxy_port = options.port_proxy)
http_client = CurlAsyncHTTPClient()
response = http_client.fetch(request_client, self.response_client)
class TornadoAsyncSolrClient(object):
def __init__(self, url):
self.url = url.strip('/')
self.update_url = self.url + '/update'
self.update_url_with_commit = self.url + '/update?commit=true'
self.select_url = self.url + '/select'
self.client = AsyncHTTPClient()
def request(self, verb, url, obj=None, callback=None):
def _callback(response):
if 200 == response.code:
callback(json.loads(response.body))
else:
raise HTTPError(response.code, response.effective_url)
request = HTTPRequest(url,
verb, {'Content-Type': 'application/json'},
json.dumps(obj) if obj else None,
use_gzip=True)
self.client.fetch(request, _callback)
def select(self, query, callback=None, **kwargs):
try:
query = query.encode('utf8')
except ValueError:
pass
kwargs['q'] = query
kwargs['wt'] = 'json'
self.request('GET',
"%s?%s" % (self.select_url, urllib.urlencode(kwargs, 1)),
callback=callback)
def add(self, callback=None, commit=False, **fields):
self.request(
'POST', self.update_url_with_commit if commit else self.update_url,
{"add": {
"doc": fields
}}, callback)
def update(self, doc_id, commit=False, callback=None, **fields):
data = {'id': doc_id}
for field, value in fields.items():
data[field] = {'set': value}
self.request(
'POST', self.update_url_with_commit if commit else self.update_url,
[data], callback)
def delete_doc(self, doc_id, callback=None):
self.request('POST', self.update_url, {"delete": {
"id": doc_id
}}, callback)
def delete_query(self, query, callback=None):
self.request('POST', self.update_url, {"delete": {
"query": query
}}, callback)
def make_zipped_content(local_config, media_type, group, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(
local_config)
logging.debug(
'preparing zip file for group "%(group)s" of remote camera %(id)s on %(url)s'
% {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config)
})
prepare_path = path + '/%(media_type)s/%(id)s/zipped/%(group)s/' % {
'media_type': media_type,
'id': camera_id,
'group': group
}
# timeout here is 100 times larger than usual - we expect a big delay
request = _make_request(scheme,
host,
port,
username,
password,
prepare_path,
timeout=100 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error(
'failed to prepare zip file for group "%(group)s" of remote camera %(id)s on %(url)s: %(msg)s'
% {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
try:
key = json.loads(response.body)['key']
except Exception as e:
logging.error(
'failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)
})
return callback(error=unicode(e))
callback({'key': key})
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def async_request(url):
future = Future()
def handle_response(response):
if response.error:
future.set_result('')
else:
future.set_result(len(response.body))
curl_client = AsyncHTTPClient()
request = HTTPRequest(url=url,headers={})
curl_client.fetch(request, handle_response)
return future
def list_media(local_config, media_type, prefix, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(
local_config)
logging.debug('getting media list for remote camera %(id)s on %(url)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config)
})
query = {}
if prefix is not None:
query['prefix'] = prefix
# timeout here is 10 times larger than usual - we expect a big delay when fetching the media list
request = _make_request(scheme,
host,
port,
username,
password,
path + '/%(media_type)s/%(id)s/list/' % {
'id': camera_id,
'media_type': media_type
},
query=query,
timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error(
'failed to get media list for remote camera %(id)s on %(url)s: %(msg)s'
% {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error(
'failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)
})
return callback(error=unicode(e))
return callback(response)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def get_media_preview(local_config, filename, media_type, width, height,
callback):
scheme, host, port, username, password, path, camera_id = _remote_params(
local_config)
logging.debug(
'getting file preview for %(filename)s of remote camera %(id)s on %(url)s'
% {
'filename': filename,
'id': camera_id,
'url': pretty_camera_url(local_config)
})
path += '/%(media_type)s/%(id)s/preview/%(filename)s' % {
'media_type': media_type,
'id': camera_id,
'filename': filename
}
query = {}
if width:
query['width'] = str(width)
if height:
query['height'] = str(height)
request = _make_request(scheme,
host,
port,
username,
password,
path,
query=query)
def on_response(response):
if response.error:
logging.error(
'failed to get file preview for %(filename)s of remote camera %(id)s on %(url)s: %(msg)s'
% {
'filename': filename,
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
callback(response.body)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def __init__(self, inqueue, outqueue, poolsize=100, **kwargs):
self.inqueue = inqueue
self.outqueue = outqueue
self.poolsize = poolsize
# self.proxyer = kwargs.get('proxyer')
self.ioloop = tornado.ioloop.IOLoop.current()
self._quit = False
self._running = False
# binding io_loop to http_client here
# self.http_client = tornado.httpclient.HTTPClient(MyCurlAsyncHTTPClient, max_clients=self.poolsize)
self.http_client = CurlAsyncHTTPClient()
def __init__(self, seed_urls=[], url_fetch_timeout=25):
ioloop.IOLoop.instance() #instanceiate IOLoop
self.client = CurlAsyncHTTPClient()
logging.basicConfig(level=logging.DEBUG,
filename="m_scrape.log",
filemode='w')
self.log = logging.getLogger(name="mod_scrape")
self.__extractors = []
#Queue Argumet urls
for url in seed_urls:
self.get(url)
def __init__(self, name, credentials, region, qualifier='$LATEST'):
"""Creates a client of AWS Lambda function with ability to invoke
it synchronously by RequestResponse invocation type.
By `synchronously` it means, that caller will receive lambda
function call result in place. Request to AWS Lambda will be made
asynchronously.
See http://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html
for details.
Usage example:
_ioloop = ioloop.IOLoop.instance()
@gen.coroutine
def async_request():
credentials = Credentials(access_key=<access_key>,
secret_key=<secret_key>)
payload = {'input_bucket': 'bucket', ...}
service = Lambda('some-service', credentials, <region>)
result = yield service(payload)
_ioloop.stop()
_ioloop.add_callback(async_request)
_ioloop.start()
:param name: Name of the AWS Lambda function.
:param credentials: AWS credentials.
:param region: AWS Lambda function region.
:param qualifier: Lambda function alias or version.
"""
self.name = name
self.region = region
self._credentials = credentials
self.client = CurlAsyncHTTPClient()
if qualifier:
query = 'Qualifier={0}'.format(quote(qualifier))
else:
query = None
self.url = URL(scheme='https',
host='lambda.{0}.amazonaws.com'.format(region),
path='{0}/functions/{1}/invocations'.format(
self.API_VERSION, name),
query=query)
self.url = str(self.url)
def get_current_picture(local_config, width, height, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(
local_config)
# logging.debug('getting current picture for remote camera %(id)s on %(url)s' % {
# 'id': camera_id,
# 'url': pretty_camera_url(local_config)})
query = {}
if width:
query['width'] = str(width)
if height:
query['height'] = str(height)
request = _make_request(scheme,
host,
port,
username,
password,
path +
'/picture/%(id)s/current/' % {'id': camera_id},
query=query)
def on_response(response):
cookies = utils.parse_cookies(response.headers.get_list('Set-Cookie'))
motion_detected = cookies.get('motion_detected_' +
str(camera_id)) == 'true'
capture_fps = cookies.get('capture_fps_' + str(camera_id))
capture_fps = float(capture_fps) if capture_fps else 0
monitor_info = cookies.get('monitor_info_' + str(camera_id))
if response.error:
logging.error(
'failed to get current picture for remote camera %(id)s on %(url)s: %(msg)s'
% {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
callback(motion_detected, capture_fps, monitor_info, response.body)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def __init__(self, server_url, io_loop=None):
"""Creates a new 'EventPublisher' object.
Events are sent in separate HTTP requests to the server given
by 'server_url'.
"""
if server_url.endswith('/publish'):
self.server_url = server_url
elif server_url.endswith('/'):
self.server_url = server_url + 'publish'
else:
self.server_url = server_url + '/publish'
self.http_client = CurlAsyncHTTPClient(io_loop=io_loop)
self.headers = {'Content-Type': mimetype_event}
self.ioloop = io_loop or tornado.ioloop.IOLoop.instance()
def __init__(self,
enable_proxy=True,
penalty_fn=None,
promote_fn=None,
max_clients=50,
before_retry_callback=None,
monitoring=False) -> None:
super().__init__()
self.shuffle_proxy_for_each_request = True
self.fetch_opts = {}
self.enable_proxy = enable_proxy
self.before_retry_callback = before_retry_callback
if monitoring:
from prometheus_client import Counter
from urllib.parse import urlparse
if not AsyncProxyClient.METRIC_RETRIED_REQUESTS:
AsyncProxyClient.METRIC_RETRIED_REQUESTS = Counter(
"async_proxy_client_retried_requests",
"Number of retried requests", ['host'])
def _before_retry_callback(*args, **kwargs):
AsyncProxyClient.METRIC_RETRIED_REQUESTS.labels(
urlparse(args[0].url).hostname).inc()
before_retry_callback and before_retry_callback(
*args, **kwargs)
self.before_retry_callback = _before_retry_callback
if self.enable_proxy:
self.proxy_manager = ProxyManager(penalty_fn, promote_fn)
self._client = CurlAsyncHTTPClient(max_clients=max_clients,
defaults=dict(validate_cert=True))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# By now, all the traitlets have been set, so we can use them to compute
# other attributes
# Use curl HTTPClient if available, else fall back to Simple one
try:
from tornado.curl_httpclient import CurlAsyncHTTPClient
self.httpclient = CurlAsyncHTTPClient(max_clients=64)
except ImportError:
from tornado.simple_httpclient import SimpleAsyncHTTPClient
self.httpclient = SimpleAsyncHTTPClient(max_clients=64)
# FIXME: Support more than just kubeconfig
self.request = request_maker()
self.pod_name = self._expand_user_properties(self.pod_name_template)
self.pvc_name = self._expand_user_properties(self.pvc_name_template)
if self.hub_connect_ip:
scheme, netloc, path, params, query, fragment = urlparse(
self.hub.api_url)
netloc = '{ip}:{port}'.format(
ip=self.hub_connect_ip,
port=self.hub_connect_port,
)
self.accessible_hub_api_url = urlunparse(
(scheme, netloc, path, params, query, fragment))
else:
self.accessible_hub_api_url = self.hub.api_url
if self.port == 0:
# Our default port is 8888
self.port = 8888
def list(local_config, callback):
scheme, host, port, username, password, path, _ = _remote_params(
local_config)
logging.debug('listing remote cameras on %(url)s' %
{'url': pretty_camera_url(local_config, camera=False)})
request = _make_request(scheme, host, port, username, password,
path + '/config/list/')
def on_response(response):
def make_camera_response(c):
return {'id': c['id'], 'name': c['name']}
if response.error:
logging.error(
'failed to list remote cameras on %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config, camera=False),
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error(
'failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config, camera=False),
'msg': unicode(e)
})
return callback(error=unicode(e))
cameras = response['cameras']
# filter out simple mjpeg cameras
cameras = [
make_camera_response(c) for c in cameras
if c['proto'] != 'mjpeg' and c.get('enabled')
]
callback(cameras)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def get_zipped_content(local_config, media_type, key, group, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(
local_config)
logging.debug('downloading zip file for remote camera %(id)s on %(url)s' %
{
'id': camera_id,
'url': pretty_camera_url(local_config)
})
request = _make_request(
scheme,
host,
port,
username,
password,
path + '/%(media_type)s/%(id)s/zipped/%(group)s/?key=%(key)s' % {
'media_type': media_type,
'group': group,
'id': camera_id,
'key': key
},
timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error(
'failed to download zip file for remote camera %(id)s on %(url)s: %(msg)s'
% {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
callback({
'data':
response.body,
'content_type':
response.headers.get('Content-Type'),
'content_disposition':
response.headers.get('Content-Disposition')
})
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def __init__(self, queue, account_id, region, access_key, secret_key):
self.queue = queue
self.account_id = account_id
self.region = region
self.session = botocore.session.Session()
self.session.set_credentials(access_key, secret_key)
self.http_client = CurlAsyncHTTPClient()
self._methods = {}
def check_timelapse_movie(local_config, group, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(
local_config)
logging.debug(
'checking timelapse movie status for remote camera %(id)s on %(url)s' %
{
'id': camera_id,
'url': pretty_camera_url(local_config)
})
request = _make_request(
scheme, host, port, username, password,
path + '/picture/%(id)s/timelapse/%(group)s/?check=true' % {
'id': camera_id,
'group': group
})
def on_response(response):
if response.error:
logging.error(
'failed to check timelapse movie status for remote camera %(id)s on %(url)s: %(msg)s'
% {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error(
'failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)
})
return callback(error=unicode(e))
callback(response)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def make_timelapse_movie(local_config, framerate, interval, group, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('making timelapse movie for group "%(group)s" of remote camera %(id)s with rate %(framerate)s/%(int)s on %(url)s' % {
'group': group or 'ungrouped',
'id': camera_id,
'framerate': framerate,
'int': interval,
'url': pretty_camera_url(local_config)})
path += '/picture/%(id)s/timelapse/%(group)s/?interval=%(int)s&framerate=%(framerate)s' % {
'id': camera_id,
'int': interval,
'framerate': framerate,
'group': group}
request = _make_request(scheme, host, port, username, password,
path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error('failed to make timelapse movie for group "%(group)s" of remote camera %(id)s with rate %(framerate)s/%(int)s on %(url)s: %(msg)s' % {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config),
'int': interval,
'framerate': framerate,
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)})
return callback(error=unicode(e))
callback(response)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def del_media_group(local_config, group, media_type, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(
local_config)
logging.debug(
'deleting group "%(group)s" of remote camera %(id)s on %(url)s' % {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config)
})
path += '/%(media_type)s/%(id)s/delete_all/%(group)s/' % {
'media_type': media_type,
'id': camera_id,
'group': group
}
request = _make_request(scheme,
host,
port,
username,
password,
path,
method='POST',
data='{}',
timeout=settings.REMOTE_REQUEST_TIMEOUT,
content_type='application/json')
def on_response(response):
if response.error:
logging.error(
'failed to delete group "%(group)s" of remote camera %(id)s on %(url)s: %(msg)s'
% {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)
})
return callback(error=utils.pretty_http_error(response))
callback()
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.root = Path(options.root).expanduser()
self.db = Mongo('dl')
self.rd = Redis()
self.email = AioEmail(smtp='smtp.exmail.qq.com', sender='*****@*****.**', user='******', pwd="xGrS"+"2Kch"[::-1]+"Hx4y95mF")
self.http = CurlAsyncHTTPClient()
self.cache = collections.defaultdict(dict)
self.config = config
def fetch(self, *args, **kwargs):
ret = []
if not len(args) > 0:
return ret
urls = self._fetching_urls(*args, **kwargs)
http = AsyncHTTPClient(self._io_loop)
i = 0
for url in urls:
callback = self._callback(args[i], **kwargs)
logging.info("start urlfetch %s" % url)
http.fetch(url, callback)
self.queue_len = self.queue_len + 1
i += 1
self._io_loop.start()
return ret
def fetch(self, *args, **kwargs):
ret = []
if not len(args) > 0:
return ret
urls = self._fetching_urls(*args, **kwargs)
http = AsyncHTTPClient()
i = 0
for url in urls:
callback = self._callback(args[i], **kwargs)
logging.info("start urlfetch %s" % url)
http.fetch(url, callback)
self.queue_len = self.queue_len + 1
i += 1
ioloop.IOLoop.instance().start()
return ret
def __init__(self,
*,
max_clients=100,
connect_timeout=20.0,
verify_ssl=True,
**kwargs):
self._httpclient = AsyncHTTPClient(max_clients=max_clients, **kwargs)
self._connect_timeout = connect_timeout
self._verify_ssl = verify_ssl
def list(local_config, callback):
scheme, host, port, username, password, path, _ = _remote_params(local_config)
logging.debug('listing remote cameras on %(url)s' % {
'url': pretty_camera_url(local_config, camera=False)})
request = _make_request(scheme, host, port, username, password,
path + '/config/list/')
def on_response(response):
def make_camera_response(c):
return {
'id': c['id'],
'name': c['name']
}
if response.error:
logging.error('failed to list remote cameras on %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config, camera=False),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config, camera=False),
'msg': unicode(e)})
return callback(error=unicode(e))
cameras = response['cameras']
# filter out simple mjpeg cameras
cameras = [make_camera_response(c) for c in cameras
if c['proto'] != 'mjpeg' and c.get('enabled')]
callback(cameras)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def __init__(self, process_count=None, seed_urls=[], seed_file=None,
process_class=PageProcessorBase, url_fetch_timeout=25, obey_robots=True,
redis={}):
self.__manager = Manager()
ioloop.IOLoop.instance() #instanceiate IOLoop
self.http_client = CurlAsyncHTTPClient()
self.page_queue = self.__manager.Queue()
self.log_queue = self.__manager.Queue()
logging.basicConfig(level=logging.DEBUG, filename="m_scrape.log", filemode='w')
self.log = logging.getLogger(name="mod_scrape")
self.process_handler = process_class
self.process_count = process_count
self.timeout = url_fetch_timeout
self.robot_info = {
"obey_robots": obey_robots,
"redis": redis
}
if self.process_count == None:
try:
self.log.log(logging.INFO, "No process argument supplied, attempting to use cpu_count")
self.process_count = cpu_count()
self.log.log(logging.INFO, "Process count set to: %s"% self.process_count)
except NotImplementedError:
self.log.log(logging.ERROR, "Method cpu_count failed, defaulting to 1")
self.process_count = 1
self.__extractors = []
#Queue File Urls
if seed_file:
read_thread = threading.Thread(target=self.read_thread, args=(seed_file,))
read_thread.daemon =True
read_thread.start()
#Queue Argumet urls
for url in seed_urls:
self.__queue_url(url)
def make_zipped_content(local_config, media_type, group, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('preparing zip file for group "%(group)s" of remote camera %(id)s on %(url)s' % {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config)})
prepare_path = path + '/%(media_type)s/%(id)s/zipped/%(group)s/' % {
'media_type': media_type,
'id': camera_id,
'group': group}
# timeout here is 100 times larger than usual - we expect a big delay
request = _make_request(scheme, host, port, username, password,
prepare_path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error('failed to prepare zip file for group "%(group)s" of remote camera %(id)s on %(url)s: %(msg)s' % {
'group': group or 'ungrouped',
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
try:
key = json.loads(response.body)['key']
except Exception as e:
logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)})
return callback(error=unicode(e))
callback({'key': key})
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def main():
parse_command_line()
app = Application([('/', ChunkHandler)])
app.listen(options.port, address='127.0.0.1')
def callback(response):
response.rethrow()
assert len(response.body) == (options.num_chunks * options.chunk_size)
logging.warning("fetch completed in %s seconds", response.request_time)
IOLoop.instance().stop()
logging.warning("Starting fetch with curl client")
curl_client = CurlAsyncHTTPClient()
curl_client.fetch('http://localhost:%d/' % options.port,
callback=callback)
IOLoop.instance().start()
logging.warning("Starting fetch with simple client")
simple_client = SimpleAsyncHTTPClient()
simple_client.fetch('http://localhost:%d/' % options.port,
callback=callback)
IOLoop.instance().start()
def main_curl():
# parse_command_line()
# app = Application([('/', ChunkHandler)])
# app.listen(options.port, address='127.0.0.1')
def callback(response):
# response.rethrow()
# assert len(response.body) == (options.num_chunks * options.chunk_size)
logging.warning("fetch completed in %s seconds", response.request_time)
logging.warning("response status: %s", response.code)
IOLoop.instance().stop()
url = 'http://localhost:%d/chunked' % options.port
logging.warning("Starting fetch with curl client from %s", url)
curl_client = CurlAsyncHTTPClient()
curl_client.fetch(request=url,
method='POST',
callback=callback,
body='testdatatestdata',
headers={'Transfer-Encoding': 'chunked',
'Expect': '100-continue'})
IOLoop.instance().start()
def list_media(local_config, media_type, prefix, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('getting media list for remote camera %(id)s on %(url)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config)})
query = {}
if prefix is not None:
query['prefix'] = prefix
# timeout here is 10 times larger than usual - we expect a big delay when fetching the media list
request = _make_request(scheme, host, port, username, password,
path + '/%(media_type)s/%(id)s/list/' % {
'id': camera_id, 'media_type': media_type}, query=query,
timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)
def on_response(response):
if response.error:
logging.error('failed to get media list for remote camera %(id)s on %(url)s: %(msg)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)})
return callback(error=unicode(e))
return callback(response)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
class CurlHTTPClientTestCase(AsyncHTTPTestCase):
def setUp(self):
super(CurlHTTPClientTestCase, self).setUp()
self.http_client = CurlAsyncHTTPClient(self.io_loop)
def get_app(self):
return Application([])
def test_prepare_curl_callback_stack_context(self):
exc_info = []
def error_handler(typ, value, tb):
exc_info.append((typ, value, tb))
self.stop()
return True
with ExceptionStackContext(error_handler):
request = HTTPRequest(self.get_url('/'),
prepare_curl_callback=lambda curl: 1 / 0)
self.http_client.fetch(request, callback=self.stop)
self.wait()
self.assertEqual(1, len(exc_info))
self.assertIs(exc_info[0][0], ZeroDivisionError)
class RT(object):
def __init__(self, base_url, max_clients, verify=True):
self.http_client = CurlAsyncHTTPClient(IOLoop.instance(), max_clients=max_clients)
self.base_url = base_url
self.verify = verify
def fetch(self, url, callback, **kwargs):
kwargs['request_timeout'] = 60
_prepare_defaults(kwargs)
kwargs['validate_cert'] = self.verify
return self.http_client.fetch('{}{}'.format(self.base_url, url), callback=callback, **kwargs)
def stream(self, url, cb, complete_cb, **kwargs):
kwargs['request_timeout'] = 60
kwargs['validate_cert'] = self.verify
_prepare_defaults(kwargs)
return self.http_client.fetch(
'{}{}'.format(self.base_url, url),
callback=complete_cb,
streaming_callback=cb,
**kwargs)
def __init__(self, seed_urls=[], url_fetch_timeout=25):
ioloop.IOLoop.instance() #instanceiate IOLoop
self.client = CurlAsyncHTTPClient()
logging.basicConfig(level=logging.DEBUG, filename="m_scrape.log", filemode='w')
self.log = logging.getLogger(name="mod_scrape")
self.__extractors = []
#Queue Argumet urls
for url in seed_urls:
self.get(url)
def get_media_preview(local_config, filename, media_type, width, height, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('getting file preview for %(filename)s of remote camera %(id)s on %(url)s' % {
'filename': filename,
'id': camera_id,
'url': pretty_camera_url(local_config)})
path += '/%(media_type)s/%(id)s/preview/%(filename)s' % {
'media_type': media_type,
'id': camera_id,
'filename': filename}
query = {}
if width:
query['width'] = str(width)
if height:
query['height'] = str(height)
request = _make_request(scheme, host, port, username, password,
path, query=query)
def on_response(response):
if response.error:
logging.error('failed to get file preview for %(filename)s of remote camera %(id)s on %(url)s: %(msg)s' % {
'filename': filename,
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
callback(response.body)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def get_current_picture(local_config, width, height, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
# logging.debug('getting current picture for remote camera %(id)s on %(url)s' % {
# 'id': camera_id,
# 'url': pretty_camera_url(local_config)})
query = {}
if width:
query['width'] = str(width)
if height:
query['height'] = str(height)
request = _make_request(scheme, host, port, username, password,
path + '/picture/%(id)s/current/' % {'id': camera_id},
query=query)
def on_response(response):
cookies = utils.parse_cookies(response.headers.get_list('Set-Cookie'))
motion_detected = cookies.get('motion_detected_' + str(camera_id)) == 'true'
capture_fps = cookies.get('capture_fps_' + str(camera_id))
capture_fps = float(capture_fps) if capture_fps else 0
monitor_info = cookies.get('monitor_info_' + str(camera_id))
if response.error:
logging.error('failed to get current picture for remote camera %(id)s on %(url)s: %(msg)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
callback(motion_detected, capture_fps, monitor_info, response.body)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def get_config(local_config, callback):
scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
logging.debug('getting config for remote camera %(id)s on %(url)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config)})
request = _make_request(scheme, host, port, username, password,
path + '/config/%(id)s/get/' % {'id': camera_id})
def on_response(response):
if response.error:
logging.error('failed to get config for remote camera %(id)s on %(url)s: %(msg)s' % {
'id': camera_id,
'url': pretty_camera_url(local_config),
'msg': utils.pretty_http_error(response)})
return callback(error=utils.pretty_http_error(response))
try:
response = json.loads(response.body)
except Exception as e:
logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
'url': pretty_camera_url(local_config),
'msg': unicode(e)})
return callback(error=unicode(e))
response['host'] = host
response['port'] = port
callback(response)
http_client = CurlAsyncHTTPClient()
http_client.fetch(request, _callback_wrapper(on_response))
def __init__(self, name, credentials, region, qualifier='$LATEST'):
"""Creates a client of AWS Lambda function with ability to invoke
it synchronously by RequestResponse invocation type.
By `synchronously` it means, that caller will receive lambda
function call result in place. Request to AWS Lambda will be made
asynchronously.
See http://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html
for details.
Usage example:
_ioloop = ioloop.IOLoop.instance()
@gen.coroutine
def async_request():
credentials = Credentials(access_key=<access_key>,
secret_key=<secret_key>)
payload = {'input_bucket': 'bucket', ...}
service = Lambda('some-service', credentials, <region>)
result = yield service(payload)
_ioloop.stop()
_ioloop.add_callback(async_request)
_ioloop.start()
:param name: Name of the AWS Lambda function.
:param credentials: AWS credentials.
:param region: AWS Lambda function region.
:param qualifier: Lambda function alias or version.
"""
self.name = name
self.region = region
self._credentials = credentials
self.client = CurlAsyncHTTPClient()
if qualifier:
query = 'Qualifier={0}'.format(quote(qualifier))
else:
query = None
self.url = URL(scheme='https',
host='lambda.{0}.amazonaws.com'.format(region),
path='{0}/functions/{1}/invocations'.format(
self.API_VERSION, name),
query=query)
self.url = str(self.url)
def __init__(self, server_url, io_loop=None):
"""Creates a new 'EventPublisher' object.
Events are sent in separate HTTP requests to the server given
by 'server_url'.
"""
if server_url.endswith('/publish'):
self.server_url = server_url
elif server_url.endswith('/'):
self.server_url = server_url + 'publish'
else:
self.server_url = server_url + '/publish'
self.http_client = CurlAsyncHTTPClient(io_loop=io_loop)
self.headers = {'Content-Type': mimetype_event}
self.ioloop = io_loop or tornado.ioloop.IOLoop.instance()
