def report_sauce_status(self, name, status, tags=[], remote_url=''):
"""Report test status and tags to SauceLabs
"""
job_id = BuiltIn().get_library_instance(
'Selenium2Library')._current_browser().session_id
if USERNAME_ACCESS_KEY.match(remote_url):
username, access_key =\
USERNAME_ACCESS_KEY.findall(remote_url)[0][1:]
else:
username = os.environ.get('SAUCE_USERNAME')
access_key = os.environ.get('SAUCE_ACCESS_KEY')
if not job_id:
return u"No Sauce job id found. Skipping..."
elif not username or not access_key:
return u"No Sauce environment variables found. Skipping..."
token = base64.encodestring('%s:%s' % (username, access_key))[:-1]
body = json.dumps({'name': name,
'passed': status == 'PASS',
'tags': tags})
connection = HTTPConnection('saucelabs.com')
connection.request('PUT', '/rest/v1/%s/jobs/%s' % (
username, job_id), body,
headers={'Authorization': 'Basic %s' % token}
)
return connection.getresponse().status
def __init__(self, url, key = None, secret = None, timeout = 5, context = None):
"""
Create a new Crossbar.io push client.
The only mandatory argument is the Push service endpoint of the Crossbar.io
instance to push to.
For signed pushes, provide authentication key and secret. If those are not
given, unsigned pushes are performed.
:param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push).
:type url: str
:param key: Optional key to use for signing requests.
:type key: str
:param secret: When using signed request, the secret corresponding to key.
:type secret: str
:param timeout: Timeout for requests.
:type timeout: int
:param context: If the HTTP bridge is running on HTTPS (that is securely over TLS),
then the context provides the SSL settings the client should use (e.g. the
certificate chain against which to verify the server certificate). This parameter
is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently
ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext
:type context: obj or None
"""
if six.PY2:
if type(url) == str:
url = six.u(url)
if type(key) == str:
key = six.u(key)
if type(secret) == str:
secret = six.u(secret)
assert(type(url) == six.text_type)
assert((key and secret) or (not key and not secret))
assert(key is None or type(key) == six.text_type)
assert(secret is None or type(secret) == six.text_type)
assert(type(timeout) == int)
if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT:
assert(context is None or isinstance(context, ssl.SSLContext))
self._seq = 1
self._key = key
self._secret = secret
self._endpoint = _parse_url(url)
self._endpoint['headers'] = {
"Content-type": "application/json",
"User-agent": "crossbarconnect-python"
}
if self._endpoint['secure']:
if not _HAS_SSL:
raise Exception("Bridge URL is using HTTPS, but Python SSL module is missing")
if _HAS_SSL_CLIENT_CONTEXT:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'], timeout = timeout, context = context)
else:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'], timeout = timeout)
else:
self._connection = HTTPConnection(self._endpoint['host'],
self._endpoint['port'], timeout = timeout)
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' %
(self.host, self.port,))
self.sock = wsgi_fake_socket(app, self.host, self.port,
script_name)
else:
HTTPConnection.connect(self)
except Exception:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
class WebcamStreamBase(VideoStream):
request_headers = {
'Accept': '*/*',
}
def __init__(self, url,
max_rate=3.0,
rate_bucket_size=None,
socket_timeout=10,
user_agent=DEFAULT_USER_AGENT):
self.url = url
netloc, self.path = _parse_url(url)
self.conn = HTTPConnection(netloc, timeout=socket_timeout)
self.request_headers = self.request_headers.copy()
self.request_headers['User-Agent'] = user_agent
self.stream = None
self.rate_limiter = BucketRateLimiter(max_rate, rate_bucket_size)
self.open_rate_limiter = BackoffRateLimiter(socket_timeout)
def __repr__(self):
return u"<%s at 0x%x: %s>" % (
self.__class__.__name__, id(self), self.url)
@classmethod
def from_settings(cls, settings, prefix='webcam.', **defaults):
config = config_from_settings(settings, prefix=prefix,
subprefix=cls.settings_subprefix,
**defaults)
return cls(**config)
def close(self):
if self.stream:
self.stream = None
if self.conn:
self.conn.close()
self.conn = None
@property
def closed(self):
return not self.conn
def next(self):
# FIXME: check closed more often?
if self.closed:
raise StopIteration()
next(self.rate_limiter)
try:
if self.stream is None:
next(self.open_rate_limiter)
self.stream = self._open_stream()
frame = next(self.stream)
self.open_rate_limiter.reset()
return frame
except Exception as ex:
self.stream = None
log.warn("Streaming failed: %s", text_type(ex) or repr(ex))
self.conn.close()
return None
def _check_storage(ipport, path):
conn = HTTPConnection(*ipport)
conn.request('GET', path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
return resp
def set_test_status(self, passed=True):
connection = HTTPConnection("saucelabs.com")
connection.request(
'PUT',
'/rest/v1/{0}/jobs/{1}'.format(
self.username, self.driver.session_id
),
json.dumps({"passed": passed}),
headers={"Authorization": "Basic {0}".format(self.sauce_auth)}
)
result = connection.getresponse()
return result.status == 200
def open_http_connection(self, code, query=None, method='GET'):
"""
Send a request to non-sandboxed Splash, return an HTTPConnection.
create_file Lua function is pre-loaded.
XXX: why can't we use requests or urllib, why
don't they close a connection after a timeout error?
"""
q = {"lua_source": self.CREATE_FILE + "\n" + code}
q.update(query or {})
conn = HTTPConnection('localhost', self.splash_unrestricted.portnum)
conn.request(method, "/execute/?" + six.moves.urllib.parse.urlencode(q))
return conn
def wait_for_server_to_hangup(ipport):
try_until = time() + 30
while True:
try:
conn = HTTPConnection(*ipport)
conn.request('GET', '/')
conn.getresponse()
except Exception:
break
if time() > try_until:
raise Exception(
'Still answering on %s:%s after 30 seconds' % ipport)
sleep(0.1)
def __init__(self, host, port=None, key_file=None, cert_file=None,
ca_certs=None, strict=None, **kwargs):
if six.PY2:
HTTPConnection.__init__(self, host=host, port=port, strict=strict, **kwargs)
else:
# python3's HTTPConnection dropped the strict argument.
HTTPConnection.__init__(self, host=host, port=port, **kwargs)
self.key_file = key_file
self.cert_file = cert_file
self.ca_certs = ca_certs
if self.ca_certs:
self.cert_reqs = ssl.CERT_REQUIRED
else:
self.cert_reqs = ssl.CERT_NONE
def is_alive(self):
"""Test that the connection is still alive.
Because the remote communication happens over HTTP we need to
make an explicit request to the remote. It is allowed for
WebDriver spec tests to not have a WebDriver session, since this
may be what is tested.
An HTTP request to an invalid path that results in a 404 is
proof enough to us that the server is alive and kicking.
"""
conn = HTTPConnection(self.server.host, self.server.port)
conn.request("HEAD", self.server.base_path + "invalid")
res = conn.getresponse()
return res.status == 404
def status(self):
succubus_status = super(AlppacaDaemon, self).status()
if succubus_status != 0:
print("succubus_status is {0}".format(str(succubus_status)))
return succubus_status
conn = HTTPConnection('169.254.169.254', timeout=0.1)
try:
conn.request("GET", "/")
conn.getresponse()
except Exception as e:
print("Error: alppaca is not reachable via IP 169.254.169.254. {0}".format(e))
return 3
else:
return 0
def kill_server(ipport, ipport2server, pids):
server, number = get_server_number(ipport, ipport2server)
err = Manager([server]).kill(number=number)
if err:
raise Exception("unable to kill %s" % (server if not number else "%s%s" % (server, number)))
try_until = time() + 30
while True:
try:
conn = HTTPConnection(*ipport)
conn.request("GET", "/")
conn.getresponse()
except Exception as err:
break
if time() > try_until:
raise Exception("Still answering on %s:%s after 30 seconds" % ipport)
sleep(0.1)
def _get_conn(self):
if self._conn is not None:
return self._conn
host, port = self._daemon.split(':', 2)
port = int(port)
self._conn = HTTPConnection(host, port, timeout=self._timeout)
return self._conn
def send_email(request):
try:
recipients = request.GET['to'].split(',')
url = request.GET['url']
proto, server, path, query, frag = urlsplit(url)
if query: path += '?' + query
conn = HTTPConnection(server)
conn.request('GET',path)
try: # Python 2.7+, use buffering of HTTP responses
resp = conn.getresponse(buffering=True)
except TypeError: # Python 2.6 and older
resp = conn.getresponse()
assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason)
rawData = resp.read()
conn.close()
message = MIMEMultipart()
message['Subject'] = "Graphite Image"
message['To'] = ', '.join(recipients)
message['From'] = 'composer@%s' % gethostname()
text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(),url) )
image = MIMEImage( rawData )
image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png"))
message.attach(text)
message.attach(image)
s = SMTP(settings.SMTP_SERVER)
s.sendmail('composer@%s' % gethostname(),recipients,message.as_string())
s.quit()
return HttpResponse( "OK" )
except:
return HttpResponse( format_exc() )
def check_server(ipport, ipport2server, pids, timeout=CHECK_SERVER_TIMEOUT):
server = ipport2server[ipport]
if server[:-1] in ('account', 'container', 'object'):
if int(server[-1]) > 4:
return None
path = '/connect/1/2'
if server[:-1] == 'container':
path += '/3'
elif server[:-1] == 'object':
path += '/3/4'
try_until = time() + timeout
while True:
try:
conn = HTTPConnection(*ipport)
conn.request('GET', path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception as err:
if time() > try_until:
print(err)
print('Giving up on %s:%s after %s seconds.' % (
server, ipport, timeout))
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth('http://%s:%d/auth/v1.0' % ipport,
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception as err:
if time() > try_until:
print(err)
print('Giving up on proxy:8080 after 30 seconds.')
raise err
sleep(0.1)
return None
def predict(self, examples, **kwargs):
"""Run prediction over HTTP/REST.
:param examples: The input examples
:return: The outcomes
"""
verify_example(examples, self.input_keys)
request = self.create_request(examples)
conn = HTTPConnection(self.hostname, self.port)
conn.request('POST', self.path, json.dumps(request), self.headers)
response = conn.getresponse().read()
outcomes_list = json.loads(response)
if "error" in outcomes_list:
raise ValueError("remote server returns error: {0}".format(outcomes_list["error"]))
outcomes_list = outcomes_list["outputs"]
outcomes_list = self.deserialize_response(examples, outcomes_list)
return outcomes_list
def check_server(ipport, ipport2server, pids, timeout=CHECK_SERVER_TIMEOUT):
server = ipport2server[ipport]
if server[:-1] in ("account", "container", "object"):
if int(server[-1]) > 4:
return None
path = "/connect/1/2"
if server[:-1] == "container":
path += "/3"
elif server[:-1] == "object":
path += "/3/4"
try_until = time() + timeout
while True:
try:
conn = HTTPConnection(*ipport)
conn.request("GET", path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception("Unexpected status %s" % resp.status)
break
except Exception as err:
if time() > try_until:
print(err)
print("Giving up on %s:%s after %s seconds." % (server, ipport, timeout))
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth("http://%s:%d/auth/v1.0" % ipport, "test:tester", "testing")
account = url.split("/")[-1]
head_account(url, token)
return url, token, account
except Exception as err:
if time() > try_until:
print(err)
print("Giving up on proxy:8080 after 30 seconds.")
raise err
sleep(0.1)
return None
def _get_conn(self):
if not self.connected:
return None
if self._conn is not None:
return self._conn
host, port = self.master.split(':', 2)
port = int(port)
self._conn = HTTPConnection(host, port, timeout=self._timeout)
return self._conn
def compute_engine_id():
"""Gets the Compute Engine project ID if it can be inferred.
Uses 169.254.169.254 for the metadata server to avoid request
latency from DNS lookup.
See https://cloud.google.com/compute/docs/metadata#metadataserver
for information about this IP address. (This IP is also used for
Amazon EC2 instances, so the metadata flavor is crucial.)
See https://github.com/google/oauth2client/issues/93 for context about
DNS latency.
:rtype: string or ``NoneType``
:returns: Compute Engine project ID if the metadata service is available,
else ``None``.
"""
host = '169.254.169.254'
uri_path = '/computeMetadata/v1/project/project-id'
headers = {'Metadata-Flavor': 'Google'}
connection = HTTPConnection(host, timeout=0.1)
try:
connection.request('GET', uri_path, headers=headers)
response = connection.getresponse()
if response.status == 200:
return response.read()
except socket.error: # socket.timeout or socket.error(64, 'Host is down')
pass
finally:
connection.close()
def __init__(self, url,
max_rate=3.0,
rate_bucket_size=None,
socket_timeout=10,
user_agent=DEFAULT_USER_AGENT):
self.url = url
netloc, self.path = _parse_url(url)
self.conn = HTTPConnection(netloc, timeout=socket_timeout)
self.request_headers = self.request_headers.copy()
self.request_headers['User-Agent'] = user_agent
self.stream = None
self.rate_limiter = BucketRateLimiter(max_rate, rate_bucket_size)
self.open_rate_limiter = BackoffRateLimiter(socket_timeout)
def test_garbage_in(self):
# Connect without SSL regardless of server.scheme
c = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
c._output(b'gjkgjklsgjklsgjkljklsg')
c._send_output()
response = c.response_class(c.sock, method='GET')
try:
response.begin()
self.assertEqual(response.status, 400)
self.assertEqual(response.fp.read(22),
b'Malformed Request-Line')
c.close()
except socket.error:
e = sys.exc_info()[1]
# "Connection reset by peer" is also acceptable.
if e.errno != errno.ECONNRESET:
raise
def export_to_myexp(self, trans, id, myexp_username, myexp_password):
"""
Exports a workflow to myExperiment website.
"""
trans.workflow_building_mode = workflow_building_modes.ENABLED
stored = self.get_stored_workflow(trans, id, check_ownership=False, check_accessible=True)
# Convert workflow to dict.
workflow_dict = self._workflow_to_dict(trans, stored)
#
# Create and submit workflow myExperiment request.
#
# Create workflow content JSON.
workflow_content = json.dumps(workflow_dict, indent=4, sort_keys=True)
# Create myExperiment request.
request_raw = trans.fill_template(
"workflow/myexp_export.mako",
workflow_name=workflow_dict['name'],
workflow_description=workflow_dict['annotation'],
workflow_content=workflow_content,
workflow_svg=self._workflow_to_svg_canvas(trans, stored).tostring()
)
# strip() b/c myExperiment XML parser doesn't allow white space before XML; utf-8 handles unicode characters.
request = unicodify(request_raw.strip(), 'utf-8')
# Do request and get result.
auth_header = base64.b64encode('%s:%s' % (myexp_username, myexp_password))
headers = {"Content-type": "text/xml", "Accept": "text/xml", "Authorization": "Basic %s" % auth_header}
myexp_url = trans.app.config.get("myexperiment_url", self.__myexp_url)
conn = HTTPConnection(myexp_url)
# NOTE: blocks web thread.
conn.request("POST", "/workflow.xml", request, headers)
response = conn.getresponse()
response_data = response.read()
conn.close()
# Do simple parse of response to see if export successful and provide user feedback.
parser = SingleTagContentsParser('id')
parser.feed(response_data)
myexp_workflow_id = parser.tag_content
workflow_list_str = " <br>Return to <a href='%s'>workflow list." % url_for(controller='workflows', action='list')
if myexp_workflow_id:
return trans.show_message(
"""Workflow '%s' successfully exported to myExperiment. <br/>
<a href="http://%s/workflows/%s">Click here to view the workflow on myExperiment</a> %s
""" % (stored.name, myexp_url, myexp_workflow_id, workflow_list_str),
use_panels=True)
else:
return trans.show_error_message(
"Workflow '%s' could not be exported to myExperiment. Error: %s %s" %
(stored.name, response_data, workflow_list_str), use_panels=True)
def test_http_over_https(self):
if self.scheme != 'https':
return self.skip('skipped (not running HTTPS)... ')
# Try connecting without SSL.
conn = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
conn.putrequest('GET', '/', skip_host=True)
conn.putheader('Host', self.HOST)
conn.endheaders()
response = conn.response_class(conn.sock, method='GET')
try:
response.begin()
self.assertEqual(response.status, 400)
self.body = response.read()
self.assertBody('The client sent a plain HTTP request, but this '
'server only speaks HTTPS on this port.')
except socket.error:
e = sys.exc_info()[1]
# "Connection reset by peer" is also acceptable.
if e.errno != errno.ECONNRESET:
raise
def _checkURL(self, url):
"""
Check if the ``url`` is *alive* (i.e., remote server returns code 200(OK)
or 401 (unauthorized)).
"""
try:
p = urlparse(url)
h = HTTPConnection(p[1])
h.putrequest('HEAD', p[2])
h.endheaders()
r = h.getresponse()
# CloudMan UI is pwd protected so include 401
if r.status in (200, 401):
return True
except Exception:
# No response or no good response
pass
return False
def _findMatchUrl(self, tag):
h3 = tag.find('h3')
if not h3:
return ''
a = h3.find('a')
url = a.attrs.get('href', '')
# decode url
host = parse.urlsplit(url).netloc
path = url[len(parse.urljoin(url, '/')) - 1:]
conn = HTTPConnection(host, timeout=10)
conn.request('GET', path)
req = conn.getresponse()
r_url = req.getheader('Location')
conn.close()
return r_url
def _request(self, method, path, body=None):
payload = None
if body is not None:
try:
payload = json.dumps(body)
except ValueError:
raise ValueError("Failed to encode request body as JSON: {}".format(
json.dumps(body, indent=2)))
if isinstance(payload, text_type):
payload = body.encode("utf-8")
conn = HTTPConnection(self.host, self.port)
try:
conn.request(method, path, payload)
yield conn.getresponse()
finally:
conn.close()
def _get_version(self, master):
if master is not None:
conn = None
host, port = master.split(':', 2)
port = int(port)
try:
conn = HTTPConnection(host, port, timeout=self._timeout)
conn.request('GET', '/version')
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
return
return json.loads(resp.read().decode('utf-8'))['version']
except Exception:
logger.exception('Error')
pass
finally:
if conn:
conn.close()
def retry_http_ntlm_sspi_auth(self, host, req, authenticatehdr,
auth_header):
url = req.full_url
scheme, _, host, path = url.split('/', 3)
h = HTTPConnection(host) if scheme == 'http:' else HTTPSConnection(
host)
headers = dict(req.unredirected_hdrs)
headers.update(
dict((k, v) for k, v in req.headers.items() if k not in headers))
headers["Connection"] = "Keep-Alive"
headers[auth_header] = "NTLM " + self.get_auth_req()
h.request(req.get_method(), req.selector, req.data, headers)
response = h.getresponse()
response.fp = None # keep-alive
ntlmauth = response.headers.get(authenticatehdr)
if ntlmauth is not None and ntlmauth.startswith('NTLM '):
challenge = ntlmauth[5:]
challenge_response = self.create_challenge_response(challenge)
if challenge_response is None:
_log.warning('Failed to authenticate using NTLM')
return None
headers["Connection"] = "Close"
headers[auth_header] = "NTLM " + challenge_response
h.request(req.get_method(), req.selector, req.data, headers)
response = h.getresponse()
return addinfourl(response, response.msg, req.get_full_url(),
response.status)
def _checkURL(self, url):
"""
Check if the ``url`` is *alive* (i.e., remote server returns code 200(OK)
or 401 (unauthorized)).
"""
try:
p = urlparse(url)
h = HTTPConnection(p[1])
h.putrequest('HEAD', p[2])
h.endheaders()
r = h.getresponse()
if r.status in (200, 401): # CloudMan UI is pwd protected so include 401
return True
except Exception:
# No response or no good response
pass
return False
def send_email(request):
try:
recipients = request.GET['to'].split(',')
url = request.GET['url']
proto, server, path, query, frag = urlsplit(url)
if query: path += '?' + query
conn = HTTPConnection(server)
conn.request('GET', path)
try: # Python 2.7+, use buffering of HTTP responses
resp = conn.getresponse(buffering=True)
except TypeError: # Python 2.6 and older
resp = conn.getresponse()
assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status,
resp.reason)
rawData = resp.read()
conn.close()
message = MIMEMultipart()
message['Subject'] = "Graphite Image"
message['To'] = ', '.join(recipients)
message['From'] = 'composer@%s' % gethostname()
text = MIMEText(
"Image generated by the following graphite URL at %s\r\n\r\n%s" %
(ctime(), url))
image = MIMEImage(rawData)
image.add_header('Content-Disposition',
'attachment',
filename="composer_" + strftime("%b%d_%I%M%p.png"))
message.attach(text)
message.attach(image)
s = SMTP(settings.SMTP_SERVER)
s.sendmail('composer@%s' % gethostname(), recipients,
message.as_string())
s.quit()
return HttpResponse("OK")
except:
return HttpResponse(format_exc())
def test_no_content_length(self):
# "The presence of a message-body in a request is signaled by the
# inclusion of a Content-Length or Transfer-Encoding header field in
# the request's message-headers."
#
# Send a message with neither header and no body. Even though
# the request is of method POST, this should be OK because we set
# request.process_request_body to False for our handler.
c = self.make_connection()
c.request('POST', '/no_body')
response = c.getresponse()
self.body = response.fp.read()
self.status = str(response.status)
self.assertStatus(200)
self.assertBody(b'Hello world!')
# Now send a message that has no Content-Length, but does send a body.
# Verify that CP times out the socket and responds
# with 411 Length Required.
if self.scheme == 'https':
c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT))
else:
c = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
# `_get_content_length` is needed for Python 3.6+
with mock.patch.object(
c,
'_get_content_length',
lambda body, method: None,
create=True):
# `_set_content_length` is needed for Python 2.7-3.5
with mock.patch.object(c, '_set_content_length', create=True):
c.request('POST', '/')
response = c.getresponse()
self.body = response.fp.read()
self.status = str(response.status)
self.assertStatus(411)
def kill_server(ipport, ipport2server):
server, number = get_server_number(ipport, ipport2server)
err = Manager([server]).kill(number=number)
if err:
raise Exception('unable to kill %s' % (server if not number else
'%s%s' % (server, number)))
try_until = time() + 30
while True:
try:
conn = HTTPConnection(*ipport)
conn.request('GET', '/')
conn.getresponse()
except Exception as err:
break
if time() > try_until:
raise Exception(
'Still answering on %s:%s after 30 seconds' % ipport)
sleep(0.1)
def _get_version(self, master):
if master is not None:
conn = None
host, port = master.split(':', 2)
port = int(port)
try:
conn = HTTPConnection(host, port, timeout=self._timeout)
conn.request('GET', '/version')
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
return
return json.loads(resp.read().decode('utf-8'))['version']
except Exception:
logger.exception('Error')
pass
finally:
if conn:
conn.close()
def _download_http_file(self, url, target_path):
'''下载http文件
:param url: HTTP路径
:type url: string
:param target_path: 下载到目标路径
:type target_path: string
'''
url0 = url
if url[:7] == 'http://':
url = url[7:]
pos = url.find('/')
host = url[:pos]
page = url[pos:]
conn = HTTPConnection(host, port=80, timeout=60) # 60秒超时
conn.request('GET', page)
res = conn.getresponse()
if res.status != 200:
raise RuntimeError('访问：%s 错误[HTTP错误码：%s]' % (url0, res.status))
data = res.read()
conn.close()
f = open(target_path, 'wb')
f.write(data)
f.close()
def request(self, sitename, path, method='GET', data=None):
url = self.base_url + '/' + sitename + path
path = '/' + sitename + path
if isinstance(data, dict):
for k in data.keys():
if data[k] is None: del data[k]
if web.config.debug:
web.ctx.infobase_req_count = 1 + web.ctx.get(
'infobase_req_count', 0)
a = time.time()
_path = path
_data = data
headers = {}
if data:
if isinstance(data, dict):
data = dict(
(web.safestr(k), web.safestr(v)) for k, v in data.items())
data = urlencode(data)
headers['Content-Type'] = 'application/x-www-form-urlencoded'
if method == 'GET':
path += '?' + data
data = None
stats.begin("infobase", path=path, method=method, data=data)
conn = HTTPConnection(self.base_url)
env = web.ctx.get('env') or {}
if self.auth_token:
c = SimpleCookie()
c['infobase_auth_token'] = quote(self.auth_token)
cookie = c.output(header='').strip()
headers['Cookie'] = cookie
# pass the remote ip to the infobase server
headers['X-REMOTE-IP'] = web.ctx.get('ip')
try:
conn.request(method, path, data, headers=headers)
response = conn.getresponse()
stats.end()
except socket.error:
stats.end(error=True)
logger.error("Unable to connect to infobase server", exc_info=True)
raise ClientException("503 Service Unavailable",
"Unable to connect to infobase server")
cookie = response.getheader('Set-Cookie')
if cookie:
c = SimpleCookie()
c.load(cookie)
if 'infobase_auth_token' in c:
auth_token = c['infobase_auth_token'].value
# The auth token will be in urlquoted form, unquote it before use.
# Otherwise, it will be quoted twice this value is set as cookie.
auth_token = auth_token and unquote(auth_token)
self.set_auth_token(auth_token)
if web.config.debug:
b = time.time()
print("%.02f (%s):" %
(round(b - a, 2), web.ctx.infobase_req_count),
response.status,
method,
_path,
_data,
file=web.debug)
if response.status == 200:
return response.read()
else:
self.handle_error("%d %s" % (response.status, response.reason),
response.read())
def stop_server(self):
"""send QUIT request to http server running on localhost:<port>"""
conn = HTTPConnection("127.0.0.1:{}".format(self.port))
conn.request("QUIT", "/")
conn.getresponse()
def _makeConnection(self, host=None, port=None):
h = HTTPConnection(host or self.LOCALHOST, port or self.port)
self.addCleanup(h.close)
return h
class MesosExecutorDriver(Process, ExecutorDriver):
def __init__(self, executor):
env = os.environ
agent_endpoint = env['MESOS_AGENT_ENDPOINT']
framework_id = env['MESOS_FRAMEWORK_ID']
assert framework_id
self.framework_id = dict(value=framework_id)
executor_id = env['MESOS_EXECUTOR_ID']
self.executor_id = dict(value=executor_id)
grace_shutdown_period = env.get('MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD')
if grace_shutdown_period:
self.grace_shutdown_period = parse_duration(grace_shutdown_period)
else:
self.grace_shutdown_period = 0.0
self.checkpoint = bool(env.get('MESOS_CHECKPOINT'))
self.local = bool(env.get('MESOS_LOCAL'))
self.executor = executor
self.framework_info = None
self.executor_info = None
self.tasks = {}
self.updates = {}
self._conn = None
super(MesosExecutorDriver, self).__init__(master=agent_endpoint)
def _delay_kill(self):
def _():
try:
time.sleep(self.grace_shutdown_period)
os.killpg(0, signal.SIGKILL)
except Exception:
logger.exception('Failed to force kill executor')
t = Thread(target=_)
t.daemon = True
t.start()
def gen_request(self):
body = json.dumps(
dict(
type='SUBSCRIBE',
framework_id=self.framework_id,
executor_id=self.executor_id,
subscribe=dict(
unacknowledged_tasks=list(self.tasks.values()),
unacknowledged_updates=list(self.updates.values()),
),
))
request = ('POST /api/v1/executor HTTP/1.1\r\nHost: %s\r\n'
'Content-Type: application/json\r\n'
'Accept: application/json\r\n'
'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % (
self.master, len(body), body)
return request.encode('utf-8')
def on_close(self):
if self._conn is not None:
self._conn.close()
self._conn = None
self.version = None
self.executor.disconnected(self)
if not self.checkpoint:
if not self.local:
self._delay_kill()
self.executor.shutdown(self)
self.abort()
def on_event(self, event):
if 'type' in event:
_type = event['type'].lower()
if _type == 'shutdown':
self.on_shutdown()
return
if _type == 'heartbeat':
return
if _type not in event:
logger.error('Missing `%s` in event %s' % (_type, event))
return
event = event[_type]
func_name = 'on_%s' % (_type, )
func = getattr(self, func_name, None)
if func is not None:
func(event)
else:
logger.error('Unknown type:%s, event:%s' % (_type, event))
else:
logger.error('Unknown event:%s' % (event, ))
def on_subscribed(self, info):
executor_info = info['executor_info']
framework_info = info['framework_info']
agent_info = info['agent_info']
assert executor_info['executor_id'] == self.executor_id
assert framework_info['id'] == self.framework_id
if self.executor_info is None or self.framework_info is None:
self.executor_info = executor_info
self.framework_info = framework_info
self.executor.registered(self, executor_info, framework_info,
agent_info)
else:
self.executor.reregistered(self, agent_info)
def on_launch(self, event):
task_info = event['task']
task_id = task_info['task_id']['value']
assert task_id not in self.tasks
self.tasks[task_id] = task_info
self.executor.launchTask(self, task_info)
def on_kill(self, event):
task_id = event['task_id']
self.executor.killTask(self, task_id)
def on_acknowledged(self, event):
task_id = event['task_id']['value']
uuid_ = uuid.UUID(bytes=a2b_base64(event['uuid']))
self.updates.pop(uuid_, None)
self.tasks.pop(task_id, None)
def on_message(self, event):
data = event['data']
self.executor.frameworkMessage(self, data)
def on_error(self, event):
message = event['message']
self.executor.error(self, message)
def on_shutdown(self):
if not self.local:
self._delay_kill()
self.executor.shutdown(self)
self.abort()
def _get_conn(self):
if not self.connected:
return None
if self._conn is not None:
return self._conn
host, port = self.master.split(':', 2)
port = int(port)
self._conn = HTTPConnection(host, port, timeout=1)
return self._conn
def _send(self, body, path='/api/v1/executor', method='POST', headers={}):
conn = self._get_conn()
if conn is None:
raise RuntimeError('Not connected yet')
if body != '':
data = json.dumps(body).encode('utf-8')
headers['Content-Type'] = 'application/json'
else:
data = ''
stream_id = self.stream_id
if stream_id:
headers['Mesos-Stream-Id'] = stream_id
try:
conn.request(method, path, body=data, headers=headers)
resp = conn.getresponse()
except Exception:
self._conn.close()
self._conn = None
raise
if resp.status < 200 or resp.status >= 300:
raise RuntimeError('Failed to send request code=%s, message=%s' %
(resp.status, resp.read()))
result = resp.read()
if not result:
return {}
try:
return json.loads(result.decode('utf-8'))
except Exception:
return {}
def sendStatusUpdate(self, status):
if 'timestamp' not in status:
status['timestamp'] = int(time.time())
if 'uuid' not in status:
status['uuid'] = b2a_base64(uuid.uuid4().bytes)
if 'source' not in status:
status['source'] = 'SOURCE_EXECUTOR'
body = dict(
type='UPDATE',
executor_id=self.executor_id,
framework_id=self.framework_id,
update=dict(status=status, ),
)
self._send(body)
def sendFrameworkMessage(self, data):
body = dict(
type='MESSAGE',
executor_id=self.executor_id,
framework_id=self.framework_id,
message=dict(data=data, ),
)
self._send(body)
def putrequest(self, *args, **kw):
self.url = args[1]
response = self.fakedata.pop(0) # get first response
self.sock = FakeSocket(response) # and set up a fake socket
output.new() # as well as an output buffer
HTTPConnection.putrequest(self, *args, **kw)
class MesosSchedulerDriver(Process, SchedulerDriver):
_timeout = 10
def __init__(self, sched, framework, master_uri,
use_addict=False, implicit_acknowledgements=True,
principal=None, secret=None, failover=False,
timeout=DAY):
super(MesosSchedulerDriver, self).__init__(timeout=timeout)
self.sched = sched
self.master_uri = master_uri
self._framework = framework
self.detector = None
self._conn = None
self.version = None
self._failover = failover
self._dict_cls = Dict if use_addict else dict
self.implicit_acknowledgements = implicit_acknowledgements
if principal is not None and secret is not None:
self._basic_credential = 'Basic %s' % (
b2a_base64(
('%s:%s' % (principal, secret)).encode('ascii')
).decode('ascii').strip()
)
else:
self._basic_credential = None
@property
def framework(self):
framework = dict(self._framework)
version = self.version and tuple(
int(n) for n in self.version.split('.')
)
capabilities = [
c for c in framework.get('capabilities', [])
if c['type'] != 'GPU_RESOURCES'
]
if version and version >= (1, 0, 0):
capabilities.append(dict(type='GPU_RESOURCES'))
if capabilities:
framework['capabilities'] = capabilities
else:
framework.pop('capabilities', None)
if 'failover_timeout' not in framework:
framework['failover_timeout'] = 100
return framework
@property
def framework_id(self):
id = self._framework.get('id')
return id and id.get('value')
@framework_id.setter
def framework_id(self, id):
self._framework['id'] = dict(value=id)
def _get_version(self, master):
if master is not None:
conn = None
host, port = master.split(':', 2)
port = int(port)
try:
conn = HTTPConnection(host, port, timeout=self._timeout)
conn.request('GET', '/version')
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
return
return json.loads(resp.read().decode('utf-8'))['version']
except Exception:
logger.exception('Error')
pass
finally:
if conn:
conn.close()
def change_master(self, master):
self.version = self._get_version(master)
super(MesosSchedulerDriver, self).change_master(master)
self._close()
def start(self):
super(MesosSchedulerDriver, self).start()
uri = self.master_uri
if uri.startswith('zk://') or uri.startswith('zoo://'):
from .detector import MasterDetector
self.detector = MasterDetector(uri[uri.index('://') + 3:], self)
self.detector.start()
else:
if ':' not in uri:
uri += ':5050'
self.change_master(uri)
def stop(self, failover=False):
with self._lock:
self._failover = failover
detector = self.detector
self.detector = None
if detector:
detector.stop()
super(MesosSchedulerDriver, self).stop()
def _shutdown(self):
if not self._failover:
try:
self._teardown()
except Exception:
logger.exception('Failed to Teardown')
def _get_conn(self):
if not self.connected:
return None
if self._conn is not None:
return self._conn
host, port = self.master.split(':', 2)
port = int(port)
self._conn = HTTPConnection(host, port, timeout=self._timeout)
return self._conn
def _send(self, body, path='/api/v1/scheduler', method='POST', headers={}):
with self._lock:
conn = self._get_conn()
if conn is None:
raise RuntimeError('Not connected yet')
if body != '':
data = json.dumps(body).encode('utf-8')
headers['Content-Type'] = 'application/json'
else:
data = b''
stream_id = self.stream_id
if stream_id:
headers['Mesos-Stream-Id'] = stream_id
if self._basic_credential:
headers['Authorization'] = self._basic_credential
try:
conn.request(method, path, body=data, headers=headers)
resp = conn.getresponse()
except Exception:
self._close()
raise
if resp.status < 200 or resp.status >= 300:
raise RuntimeError('Failed to send request %s: %s\n%s' % (
resp.status, resp.read(), data))
result = resp.read()
if not result:
return {}
try:
return json.loads(result.decode('utf-8'))
except Exception:
return {}
def _teardown(self):
if self.connected:
framework_id = self.framework_id
if framework_id:
self._send(dict(
type='TEARDOWN',
framework_id=dict(
value=framework_id,
),
))
self._framework.pop('id', None)
def acceptOffers(self, offer_ids, operations, filters=None):
if not operations:
return self.declineOffer(offer_ids, filters=filters)
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
accept = dict(
offer_ids=[offer_ids]
if isinstance(offer_ids, dict)
else offer_ids,
operations=operations,
)
if filters is not None:
accept['filters'] = filters
body = dict(
type='ACCEPT',
framework_id=dict(
value=framework_id,
),
accept=accept,
)
self._send(body)
def acceptInverseOffers(self, offer_ids, filters=None):
framework_id = self.framework_id
assert framework_id
accept_inverse_offers = dict(
inverse_offer_ids=[offer_ids]
if isinstance(offer_ids, dict)
else offer_ids
)
if filters is not None:
accept_inverse_offers['filters'] = filters
body = dict(
type='ACCEPT_INVERSE_OFFERS',
framework_id=dict(
value=framework_id,
),
accept_inverse_offers=accept_inverse_offers,
)
self._send(body)
def launchTasks(self, offer_ids, tasks, filters=None):
if not tasks:
return self.declineOffer(offer_ids, filters=filters)
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
operations = [dict(
type='LAUNCH',
launch=dict(
task_infos=tasks
),
)]
self.acceptOffers(offer_ids, operations, filters=filters)
def declineOffer(self, offer_ids, filters=None):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
decline = dict(
offer_ids=[offer_ids] if isinstance(offer_ids, dict) else offer_ids
)
if filters is not None:
decline['filters'] = filters
body = dict(
type='DECLINE',
framework_id=dict(
value=framework_id,
),
decline=decline,
)
self._send(body)
def declineInverseOffer(self, offer_ids, filters=None):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
decline_inverse_offers = dict(
inverse_offer_ids=[offer_ids]
if isinstance(offer_ids, dict)
else offer_ids
)
if filters is not None:
decline_inverse_offers['filters'] = filters
body = dict(
type='DECLINE_INVERSE_OFFERS',
framework_id=dict(
value=framework_id,
),
decline_inverse_offers=decline_inverse_offers,
)
self._send(body)
def reviveOffers(self, roles=()):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='REVIVE',
framework_id=dict(
value=framework_id,
),
)
if roles:
body['revive'] = dict(roles=list(roles))
self._send(body)
def suppressOffers(self, roles=()):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='SUPPRESS',
framework_id=dict(
value=framework_id,
),
)
if roles:
body['suppress'] = dict(roles=list(roles))
self._send(body)
def killTask(self, task_id):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='KILL',
framework_id=dict(
value=framework_id,
),
kill=dict(
task_id=task_id,
),
)
self._send(body)
def acknowledgeStatusUpdate(self, status):
if self.connected and 'uuid' in status:
framework_id = self.framework_id
assert framework_id
acknowledge = dict()
acknowledge['agent_id'] = status['agent_id']
acknowledge['task_id'] = status['task_id']
acknowledge['uuid'] = status['uuid']
body = dict(
type='ACKNOWLEDGE',
framework_id=dict(
value=framework_id,
),
acknowledge=acknowledge,
)
self._send(body)
def acknowledgeOperationStatusUpdate(self, status):
if self.connected and 'uuid' in status and 'operation_id' in status:
framework_id = self.framework_id
assert framework_id
body = dict(
type='ACKNOWLEDGE_OPERATION_STATUS',
framework_id=dict(
value=framework_id,
),
acknowledge_operation_status=status,
)
self._send(body)
def reconcileTasks(self, tasks):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='RECONCILE',
framework_id=dict(
value=framework_id,
),
reconcile=dict(
tasks=[dict(task_id=task['task_id']) for task in tasks],
),
)
self._send(body)
def reconcileOperations(self, operations_):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
operations = []
for op_ in operations_:
op = dict(
operation_id=op_['operation_id']
)
if 'agent_id' in op_:
op['agent_id'] = op_['agent_id']
if 'resource_provider_id' in op_:
op['resource_provider_id'] = op_['resource_provider_id']
operations.append(op)
body = dict(
type='RECONCILE_OPERATIONS',
framework_id=dict(
value=framework_id,
),
reconcile_operations=dict(
operations=operations,
),
)
self._send(body)
def sendFrameworkMessage(self, executor_id, agent_id, data):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
message = dict(
agent_id=agent_id,
executor_id=executor_id,
data=data,
)
body = dict(
type='MESSAGE',
framework_id=dict(
value=framework_id,
),
message=message,
)
self._send(body)
def requestResources(self, requests):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='REQUEST',
framework_id=dict(
value=framework_id,
),
request=dict(
requests=requests,
),
)
self._send(body)
def onNewMasterDetectedMessage(self, data):
master = None
try:
if isinstance(data, six.binary_type):
data = data.decode('utf-8')
parsed = json.loads(data)
if parsed and "address" in parsed:
ip = parsed["address"].get("ip")
port = parsed["address"].get("port")
if ip and port:
master = "%s:%s" % (ip, port)
except Exception:
logger.exception("No JSON content, probably connecting "
"to older Mesos version.")
if master:
self.change_master(master)
def onNoMasterDetectedMessage(self):
self.change_master(None)
def gen_request(self):
request = dict(
type='SUBSCRIBE',
subscribe=dict(
framework_info=self.framework
),
)
if 'id' in self._framework:
request['framework_id'] = self._framework['id']
data = json.dumps(request)
_authorization = ''
if self._basic_credential is not None:
_authorization = 'Authorization: %s\r\n' % (
self._basic_credential,
)
request = ('POST /api/v1/scheduler HTTP/1.1\r\nHost: %s\r\n'
'Content-Type: application/json\r\n'
'Accept: application/json\r\n%s'
'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % (
self.master, _authorization, len(data), data
)
return request.encode('utf-8')
def _close(self):
if self._conn is not None:
self._conn.close()
self._conn = None
def on_close(self):
self._close()
self.sched.disconnected(self)
def on_subscribed(self, info):
reregistered = (self.framework_id is not None)
self.framework_id = info['framework_id']['value']
hostname, port = self.master.split(':', 2)
port = int(port)
master_info = dict(
hostname=hostname,
port=port,
)
if self.version:
master_info['version'] = self.version
elif 'master_info' in info and 'version' in info['master_info']:
master_info['version'] = info['master_info']['version']
if reregistered:
self.sched.reregistered(self, self._dict_cls(master_info))
else:
framework_id = dict(
value=self.framework_id
)
self.sched.registered(
self, self._dict_cls(framework_id),
self._dict_cls(master_info)
)
def on_offers(self, event):
offers = event.get('offers', [])
if offers:
self.sched.resourceOffers(
self, [self._dict_cls(offer) for offer in offers]
)
version = self.version and tuple(
int(n) for n in self.version.split('.')
)
if not (version and version >= (1, 0, 0)):
self.on_inverse_offers(event)
def on_inverse_offers(self, event):
inverse_offers = event.get('inverse_offers', [])
if inverse_offers:
self.sched.inverseOffers(
self, [self._dict_cls(offer) for offer in inverse_offers]
)
def on_rescind(self, event):
offer_id = event['offer_id']
self.sched.offerRescinded(self, self._dict_cls(offer_id))
def on_rescind_inverse_offer(self, event):
inverse_offer_id = event['inverse_offer_id']
self.sched.inverseOfferRescinded(
self, self._dict_cls(inverse_offer_id)
)
def on_update(self, event):
status = event['status']
self.sched.statusUpdate(self, self._dict_cls(status))
if self.implicit_acknowledgements:
self.acknowledgeStatusUpdate(status)
def on_update_operation_status(self, event):
status = event['status']
self.sched.operationStatusUpdate(self, self._dict_cls(status))
if self.implicit_acknowledgements:
self.acknowledgeOperationStatusUpdate(status)
def on_message(self, message):
executor_id = message['executor_id']
agent_id = message['agent_id']
data = message['data']
self.sched.frameworkMessage(
self, self._dict_cls(executor_id), self._dict_cls(agent_id), data
)
def on_failure(self, failure):
agent_id = failure['agent_id']
if 'executor_id' not in failure:
self.sched.slaveLost(self, self._dict_cls(agent_id))
else:
self.sched.executorLost(
self, self._dict_cls(failure['executor_id']),
self._dict_cls(agent_id), failure['status']
)
def on_error(self, event):
message = event['message']
self.sched.error(self, message)
def on_heartbeat(self):
self.sched.processHeartBeat()
def on_event(self, event):
if 'type' in event:
_type = event['type'].lower()
if _type == 'heartbeat':
self.on_heartbeat()
return
if _type not in event:
logger.error(
'Missing `%s` in event %s' %
(_type, event))
return
event = event[_type]
func_name = 'on_%s' % (_type,)
func = getattr(self, func_name, None)
if func is not None:
func(event)
else:
logger.error('Unknown type:%s, event:%s' % (_type, event))
else:
logger.error('Unknown event:%s' % (event,))
class MesosExecutorDriver(Process, ExecutorDriver):
_timeout = 10
def __init__(self, executor, use_addict=False):
env = os.environ
agent_endpoint = env['MESOS_AGENT_ENDPOINT']
super(MesosExecutorDriver, self).__init__(master=agent_endpoint)
framework_id = env['MESOS_FRAMEWORK_ID']
assert framework_id
self.framework_id = dict(value=framework_id)
executor_id = env['MESOS_EXECUTOR_ID']
self.executor_id = dict(value=executor_id)
grace_shutdown_period = env.get('MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD')
if grace_shutdown_period:
self.grace_shutdown_period = parse_duration(grace_shutdown_period)
else:
self.grace_shutdown_period = 0.0
self.checkpoint = bool(env.get('MESOS_CHECKPOINT'))
self.local = bool(env.get('MESOS_LOCAL'))
self.executor = executor
self.framework_info = None
self.executor_info = None
self.tasks = {}
self.updates = {}
self._conn = None
self._dict_cls = Dict if use_addict else dict
def _delay_kill(self):
def _():
try:
time.sleep(self.grace_shutdown_period)
os.killpg(0, signal.SIGKILL)
except Exception:
logger.exception('Failed to force kill executor')
t = Thread(target=_)
t.daemon = True
t.start()
def gen_request(self):
body = json.dumps(dict(
type='SUBSCRIBE',
framework_id=self.framework_id,
executor_id=self.executor_id,
subscribe=dict(
unacknowledged_tasks=list(self.tasks.values()),
unacknowledged_updates=list(self.updates.values()),
),
))
request = ('POST /api/v1/executor HTTP/1.1\r\nHost: %s\r\n'
'Content-Type: application/json\r\n'
'Accept: application/json\r\n'
'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % (
self.master, len(body), body
)
return request.encode('utf-8')
def on_close(self):
if self._conn is not None:
self._conn.close()
self._conn = None
self.version = None
self.executor.disconnected(self)
if not self.checkpoint:
if not self.local:
self._delay_kill()
self.executor.shutdown(self)
self.abort()
def on_event(self, event):
if 'type' in event:
_type = event['type'].lower()
if _type == 'shutdown':
self.on_shutdown()
return
if _type == 'heartbeat':
return
if _type not in event:
logger.error(
'Missing `%s` in event %s' %
(_type, event))
return
event = event[_type]
func_name = 'on_%s' % (_type,)
func = getattr(self, func_name, None)
if func is not None:
func(event)
else:
logger.error('Unknown type:%s, event:%s' % (_type, event))
else:
logger.error('Unknown event:%s' % (event,))
def on_subscribed(self, info):
executor_info = info['executor_info']
framework_info = info['framework_info']
agent_info = info['agent_info']
assert executor_info['executor_id'] == self.executor_id
assert framework_info['id'] == self.framework_id
if self.executor_info is None or self.framework_info is None:
self.executor_info = executor_info
self.framework_info = framework_info
self.executor.registered(
self, self._dict_cls(executor_info),
self._dict_cls(framework_info), self._dict_cls(agent_info)
)
else:
self.executor.reregistered(self, self._dict_cls(agent_info))
def on_launch(self, event):
task_info = event['task']
task_id = task_info['task_id']['value']
assert task_id not in self.tasks
self.tasks[task_id] = task_info
self.executor.launchTask(self, self._dict_cls(task_info))
def on_kill(self, event):
task_id = event['task_id']
self.executor.killTask(self, self._dict_cls(task_id))
def on_acknowledged(self, event):
task_id = event['task_id']['value']
uuid_ = uuid.UUID(bytes=decode_data(event['uuid']))
self.updates.pop(uuid_, None)
self.tasks.pop(task_id, None)
def on_message(self, event):
data = event['data']
self.executor.frameworkMessage(self, data)
def on_error(self, event):
message = event['message']
self.executor.error(self, message)
def on_shutdown(self):
if not self.local:
self._delay_kill()
self.executor.shutdown(self)
self.abort()
def _get_conn(self):
if not self.connected:
return None
if self._conn is not None:
return self._conn
host, port = self.master.split(':', 2)
port = int(port)
self._conn = HTTPConnection(host, port, timeout=self._timeout)
return self._conn
def _send(self, body, path='/api/v1/executor', method='POST', headers={}):
with self._lock:
conn = self._get_conn()
if conn is None:
raise RuntimeError('Not connected yet')
if body != '':
data = json.dumps(body).encode('utf-8')
headers['Content-Type'] = 'application/json'
else:
data = ''
stream_id = self.stream_id
if stream_id:
headers['Mesos-Stream-Id'] = stream_id
try:
conn.request(method, path, body=data, headers=headers)
resp = conn.getresponse()
except Exception:
self._conn.close()
self._conn = None
raise
if resp.status < 200 or resp.status >= 300:
raise RuntimeError(
'Failed to send request code=%s, message=%s' % (
resp.status, resp.read()
)
)
result = resp.read()
if not result:
return {}
try:
return json.loads(result.decode('utf-8'))
except Exception:
return {}
def sendStatusUpdate(self, status):
if 'timestamp' not in status:
status['timestamp'] = int(time.time())
if 'uuid' not in status:
status['uuid'] = encode_data(uuid.uuid4().bytes)
if 'source' not in status:
status['source'] = 'SOURCE_EXECUTOR'
body = dict(
type='UPDATE',
executor_id=self.executor_id,
framework_id=self.framework_id,
update=dict(
status=status,
),
)
self._send(body)
def sendFrameworkMessage(self, data):
body = dict(
type='MESSAGE',
executor_id=self.executor_id,
framework_id=self.framework_id,
message=dict(
data=data,
),
)
self._send(body)
class HTTPWireProtocol(object):
"""
Transports messages (commands and responses) over the WebDriver
wire protocol.
Complex objects, such as ``webdriver.Element``, ``webdriver.Frame``,
and ``webdriver.Window`` are by default not marshaled to enable
use of `session.transport.send` in WPT tests::
session = webdriver.Session("127.0.0.1", 4444)
response = transport.send("GET", "element/active", None)
print response.body["value"]
# => {u'element-6066-11e4-a52e-4f735466cecf': u'<uuid>'}
Automatic marshaling is provided by ``webdriver.protocol.Encoder``
and ``webdriver.protocol.Decoder``, which can be passed in to
``HTTPWireProtocol.send`` along with a reference to the current
``webdriver.Session``::
session = webdriver.Session("127.0.0.1", 4444)
response = transport.send("GET", "element/active", None,
encoder=protocol.Encoder, decoder=protocol.Decoder,
session=session)
print response.body["value"]
# => webdriver.Element
"""
def __init__(self, host, port, url_prefix="/"):
"""
Construct interface for communicating with the remote server.
:param url: URL of remote WebDriver server.
:param wait: Duration to wait for remote to appear.
"""
self.host = host
self.port = port
self.url_prefix = url_prefix
self._conn = None
self._last_request_is_blocked = False
def __del__(self):
self.close()
def close(self):
"""Closes the current HTTP connection, if there is one."""
if self._conn:
self._conn.close()
@property
def connection(self):
"""Gets the current HTTP connection, or lazily creates one."""
if not self._conn:
conn_kwargs = {}
if not PY3:
conn_kwargs["strict"] = True
# We are not setting an HTTP timeout other than the default when the
# connection its created. The send method has a timeout value if needed.
self._conn = HTTPConnection(self.host, self.port, **conn_kwargs)
return self._conn
def url(self, suffix):
"""
From the relative path to a command end-point,
craft a full URL suitable to be used in a request to the HTTPD.
"""
return urlparse.urljoin(self.url_prefix, suffix)
def send(self,
method,
uri,
body=None,
headers=None,
encoder=json.JSONEncoder,
decoder=json.JSONDecoder,
timeout=None,
**codec_kwargs):
"""
Send a command to the remote.
The request `body` must be JSON serialisable unless a
custom `encoder` has been provided. This means complex
objects such as ``webdriver.Element``, ``webdriver.Frame``,
and `webdriver.Window`` are not automatically made
into JSON. This behaviour is, however, provided by
``webdriver.protocol.Encoder``, should you want it.
Similarly, the response body is returned au natural
as plain JSON unless a `decoder` that converts web
element references to ``webdriver.Element`` is provided.
Use ``webdriver.protocol.Decoder`` to achieve this behaviour.
The client will attempt to use persistent HTTP connections.
:param method: `GET`, `POST`, or `DELETE`.
:param uri: Relative endpoint of the requests URL path.
:param body: Body of the request. Defaults to an empty
dictionary if ``method`` is `POST`.
:param headers: Additional dictionary of headers to include
in the request.
:param encoder: JSON encoder class, which defaults to
``json.JSONEncoder`` unless specified.
:param decoder: JSON decoder class, which defaults to
``json.JSONDecoder`` unless specified.
:param codec_kwargs: Surplus arguments passed on to `encoder`
and `decoder` on construction.
:return: Instance of ``webdriver.transport.Response``
describing the HTTP response received from the remote end.
:raises ValueError: If `body` or the response body are not
JSON serialisable.
"""
if body is None and method == "POST":
body = {}
payload = None
if body is not None:
try:
payload = json.dumps(body, cls=encoder, **codec_kwargs)
except ValueError:
raise ValueError("Failed to encode request body as JSON:\n"
"%s" % json.dumps(body, indent=2))
# When the timeout triggers, the TestRunnerManager thread will reuse
# this connection to check if the WebDriver its alive and we may end
# raising an httplib.CannotSendRequest exception if the WebDriver is
# not responding and this httplib.request() call is blocked on the
# runner thread. We use the boolean below to check for that and restart
# the connection in that case.
self._last_request_is_blocked = True
response = self._request(method, uri, payload, headers, timeout=None)
self._last_request_is_blocked = False
return Response.from_http(response, decoder=decoder, **codec_kwargs)
def _request(self, method, uri, payload, headers=None, timeout=None):
if isinstance(payload, text_type):
payload = payload.encode("utf-8")
if headers is None:
headers = {}
headers.update({"Connection": "keep-alive"})
url = self.url(uri)
if self._last_request_is_blocked or self._has_unread_data():
self.close()
self.connection.request(method, url, payload, headers)
# timeout for request has to be set just before calling httplib.getresponse()
# and the previous value restored just after that, even on exception raised
try:
if timeout:
previous_timeout = self._conn.gettimeout()
self._conn.settimeout(timeout)
response = self.connection.getresponse()
finally:
if timeout:
self._conn.settimeout(previous_timeout)
return response
def _has_unread_data(self):
return self._conn and self._conn.sock and select.select([self._conn.sock], [], [], 0)[0]
class WebcamStreamBase(VideoStream):
request_headers = {
'Accept': '*/*',
}
def __init__(self,
url,
max_rate=3.0,
rate_bucket_size=None,
socket_timeout=10,
user_agent=DEFAULT_USER_AGENT):
self.url = url
netloc, self.path = _parse_url(url)
self.conn = HTTPConnection(netloc, timeout=socket_timeout)
self.request_headers = self.request_headers.copy()
self.request_headers['User-Agent'] = user_agent
self.stream = None
self.rate_limiter = BucketRateLimiter(max_rate, rate_bucket_size)
self.open_rate_limiter = BackoffRateLimiter(socket_timeout)
def __repr__(self):
return u"<%s at 0x%x: %s>" % (self.__class__.__name__, id(self),
self.url)
@classmethod
def from_settings(cls, settings, prefix='webcam.', **defaults):
config = config_from_settings(settings,
prefix=prefix,
subprefix=cls.settings_subprefix,
**defaults)
return cls(**config)
def close(self):
if self.stream:
self.stream = None
if self.conn:
self.conn.close()
self.conn = None
@property
def closed(self):
return not self.conn
def next(self):
# FIXME: check closed more often?
if self.closed:
raise StopIteration()
next(self.rate_limiter)
try:
if self.stream is None:
next(self.open_rate_limiter)
self.stream = self._open_stream()
frame = next(self.stream)
self.open_rate_limiter.reset()
return frame
except Exception as ex:
self.stream = None
log.warn("Streaming failed: %s", text_type(ex) or repr(ex))
self.conn.close()
return None
def export_to_myexp(self, trans, id, myexp_username, myexp_password):
"""
Exports a workflow to myExperiment website.
"""
trans.workflow_building_mode = workflow_building_modes.ENABLED
stored = self.get_stored_workflow(trans,
id,
check_ownership=False,
check_accessible=True)
# Convert workflow to dict.
workflow_dict = self._workflow_to_dict(trans, stored)
#
# Create and submit workflow myExperiment request.
#
# Create workflow content JSON.
workflow_content = json.dumps(workflow_dict, indent=4, sort_keys=True)
# Create myExperiment request.
request_raw = trans.fill_template(
"workflow/myexp_export.mako",
workflow_name=workflow_dict['name'],
workflow_description=workflow_dict['annotation'],
workflow_content=workflow_content,
workflow_svg=self._workflow_to_svg_canvas(trans,
stored).tostring())
# strip() b/c myExperiment XML parser doesn't allow white space before XML; utf-8 handles unicode characters.
request = unicodify(request_raw.strip(), 'utf-8')
# Do request and get result.
auth_header = base64.b64encode('%s:%s' %
(myexp_username, myexp_password))
headers = {
"Content-type": "text/xml",
"Accept": "text/xml",
"Authorization": "Basic %s" % auth_header
}
myexp_url = trans.app.config.get("myexperiment_url", self.__myexp_url)
conn = HTTPConnection(myexp_url)
# NOTE: blocks web thread.
conn.request("POST", "/workflow.xml", request, headers)
response = conn.getresponse()
response_data = response.read()
conn.close()
# Do simple parse of response to see if export successful and provide user feedback.
parser = SingleTagContentsParser('id')
parser.feed(response_data)
myexp_workflow_id = parser.tag_content
workflow_list_str = " <br>Return to <a href='%s'>workflow list." % url_for(
controller='workflows', action='list')
if myexp_workflow_id:
return trans.show_message(
"""Workflow '%s' successfully exported to myExperiment. <br/>
<a href="http://%s/workflows/%s">Click here to view the workflow on myExperiment</a> %s
""" %
(stored.name, myexp_url, myexp_workflow_id, workflow_list_str),
use_panels=True)
else:
return trans.show_error_message(
"Workflow '%s' could not be exported to myExperiment. Error: %s %s"
% (stored.name, response_data, workflow_list_str),
use_panels=True)
class MesosOperatorDaemonDriver(OperatorDaemonDriver):
_timeout = 10
def __init__(self, daemon_uri):
self.init(daemon_uri)
def init(self, daemon_uri):
"""
:param daemon_uri: masterHost:5050 or agentHost:5051
"""
self._daemon = daemon_uri
self._conn = None
def _get_conn(self):
if self._conn is not None:
return self._conn
host, port = self._daemon.split(':', 2)
port = int(port)
self._conn = HTTPConnection(host, port, timeout=self._timeout)
return self._conn
def _send(self, body, path='/api/v1/operator', method='POST', headers={}):
with self._lock:
conn = self._get_conn()
if conn is None:
raise RuntimeError('Not connected yet')
if body != '':
data = json.dumps(body).encode('utf-8')
headers['Content-Type'] = 'application/json'
else:
data = ''
# stream_id = self.stream_id
# if stream_id:
# headers['Mesos-Stream-Id'] = stream_id
try:
conn.request(method, path, body=data, headers=headers)
resp = conn.getresponse()
except Exception:
self._conn.close()
self._conn = None
raise
if resp.status < 200 or resp.status >= 300:
raise RuntimeError(
'Failed to send request code=%s, message=%s' % (
resp.status, resp.read()
)
)
result = resp.read()
if not result:
return {}
try:
return json.loads(result.decode('utf-8'))
except Exception:
return {}
def getHealth(self):
body = dict(
type='GET_HEALTH',
)
return self._send(body)
def getFlags(self):
body = dict(
type='GET_FLAGS',
)
return self._send(body)
def getVersion(self):
body = dict(
type='GET_VERSION',
)
return self._send(body)
def getMetrics(self, timeout):
body = dict(
type='GET_METRICS',
get_metrics=dict(
timeout=dict(
nanoseconds=timeout,
)
)
)
return self._send(body)
def getLoggingLevel(self):
body = dict(
type='GET_LOGGING_LEVEL',
)
return self._send(body)
def setLoggingLevel(self, level, duration):
body = dict(
type='SET_LOGGING_LEVEL',
set_logging_level=dict(
duration=dict(
nanoseconds=duration,
),
level=level,
)
)
return self._send(body)
def listFiles(self, path):
body = dict(
type='LIST_FILES',
list_files=dict(
path=path,
),
)
return self._send(body)
def readFile(self, path, length, offset):
body = dict(
type='READ_FILE',
read_file=dict(
length=length,
offset=offset,
path=path,
)
)
return self._send(body)
def getState(self):
body = dict(
type='GET_STATE',
)
return self._send(body)
def getFrameworks(self):
body = dict(
type='GET_FRAMEWORKS',
)
return self._send(body)
def getExecutors(self):
body = dict(
type='GET_EXECUTORS',
)
return self._send(body)
def getTasks(self):
body = dict(
type='GET_TASKS',
)
return self._send(body)
class MesosSchedulerDriver(Process, SchedulerDriver):
def __init__(self, sched, framework, master_uri):
super(MesosSchedulerDriver, self).__init__()
self.sched = sched
self.master_uri = master_uri
self._framework = framework
self.detector = None
self._conn = None
self.version = None
@property
def framework(self):
framework = dict(self._framework)
version = self.version and tuple(
int(n) for n in self.version.split('.'))
capabilities = [
c for c in framework.get('capabilities', [])
if c['type'] != 'GPU_RESOURCES'
]
if version and version >= (1, 0, 0):
capabilities.append(dict(type='GPU_RESOURCES'))
if capabilities:
framework['capabilities'] = capabilities
else:
framework.pop('capabilities', None)
return framework
@property
def framework_id(self):
return self._framework.get('id')
@framework_id.setter
def framework_id(self, id):
self._framework['id'] = id
def _get_version(self, master):
if master is not None:
conn = None
host, port = master.split(':', 2)
port = int(port)
try:
conn = HTTPConnection(host, port, timeout=1)
conn.request('GET', '/version')
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
return
return json.loads(resp.read())['version']
except Exception:
logger.exception('Error')
pass
finally:
if conn:
conn.close()
def change_master(self, master):
self.version = self._get_version(master)
super(MesosSchedulerDriver, self).change_master(master)
if self._conn is not None:
self._conn.close()
self._conn = None
def start(self):
super(MesosSchedulerDriver, self).start()
uri = self.master_uri
if uri.startswith('zk://') or uri.startswith('zoo://'):
from .detector import MasterDetector
self.detector = MasterDetector(uri[uri.index('://') + 3:], self)
self.detector.start()
else:
if ':' not in uri:
uri += ':5050'
self.change_master(uri)
def stop(self, failover=False):
if not failover:
try:
self._teardown()
except Exception:
logger.exception('Failed to Teardown')
if self.detector:
self.detector.stop()
super(MesosSchedulerDriver, self).stop()
def _get_conn(self):
if not self.connected:
return None
if self._conn is not None:
return self._conn
host, port = self.master.split(':', 2)
port = int(port)
self._conn = HTTPConnection(host, port, timeout=1)
return self._conn
def _send(self, body, path='/api/v1/scheduler', method='POST', headers={}):
conn = self._get_conn()
if conn is None:
raise RuntimeError('Not connected yet')
if body != '':
data = json.dumps(body).encode('utf-8')
headers['Content-Type'] = 'application/json'
else:
data = ''
stream_id = self.stream_id
if stream_id:
headers['Mesos-Stream-Id'] = stream_id
try:
conn.request(method, path, body=data, headers=headers)
resp = conn.getresponse()
except Exception:
self._conn.close()
self._conn = None
raise
if resp.status < 200 or resp.status >= 300:
raise RuntimeError('Failed to send request %s: %s\n%s' %
(resp.status, resp.read(), data))
result = resp.read()
if not result:
return {}
try:
return json.loads(result.decode('utf-8'))
except Exception:
return {}
def _teardown(self):
framework_id = self.framework_id
if framework_id:
self._send(
dict(
type='TEARDOWN',
framework_id=dict(value=framework_id, ),
))
def acceptOffers(self, offer_ids, operations, filters=None):
if not operations:
return self.declineOffer(offer_ids, filters=filters)
framework_id = self.framework_id
assert framework_id
accept = dict(
offer_ids=offer_ids,
operations=operations,
)
if filters is not None:
accept['filters'] = filters
body = dict(
type='ACCEPT',
framework_id=dict(value=framework_id, ),
accept=accept,
)
self._send(body)
def launchTasks(self, offer_ids, tasks, filters=None):
if not tasks:
return self.declineOffer(offer_ids, filters=filters)
framework_id = self.framework_id
assert framework_id
operations = [dict(
type='LAUNCH',
launch=dict(task_infos=tasks),
)]
self.acceptOffers(offer_ids, operations, filters=filters)
def declineOffer(self, offer_ids, filters=None):
framework_id = self.framework_id
assert framework_id
decline = dict(offer_ids=offer_ids, )
if filters is not None:
decline['filters'] = filters
body = dict(
type='DECLINE',
framework_id=dict(value=framework_id, ),
decline=decline,
)
self._send(body)
def reviveOffers(self):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='REVIVE',
framework_id=dict(value=framework_id, ),
)
self._send(body)
def killTask(self, task_id):
framework_id = self.framework_id
assert framework_id
body = dict(
type='KILL',
framework_id=dict(value=framework_id, ),
kill=dict(task_id=task_id, ),
)
self._send(body)
def acknowledgeStatusUpdate(self, status):
if 'uuid' in status:
framework_id = self.framework_id
assert framework_id
acknowledge = dict()
acknowledge['agent_id'] = status['agent_id']
acknowledge['task_id'] = status['task_id']
acknowledge['uuid'] = status['uuid']
body = dict(
type='ACKNOWLEDGE',
framework_id=dict(value=framework_id, ),
acknowledge=acknowledge,
)
self._send(body)
def reconcileTasks(self, tasks):
framework_id = self.framework_id
assert framework_id
body = dict(
type='RECONCILE',
framework_id=dict(value=framework_id, ),
reconcile=dict(
tasks=[dict(task_id=task['task_id']) for task in tasks], ),
)
self._send(body)
def sendFrameworkMessage(self, executor_id, agent_id, data):
framework_id = self.framework_id
assert framework_id
message = dict(
agent_id=agent_id,
executor_id=executor_id,
data=b2a_base64(data.encode('utf-8')).rstrip(),
)
body = dict(
type='MESSAGE',
framework_id=dict(value=framework_id, ),
message=message,
)
self._send(body)
def requestResources(self, requests):
framework_id = self.framework_id
assert framework_id
body = dict(
type='REQUEST',
framework_id=dict(value=framework_id, ),
request=dict(requests=requests, ),
)
self._send(body)
def onNewMasterDetectedMessage(self, data):
master = None
try:
parsed = json.loads(data)
if parsed and "address" in parsed:
ip = parsed["address"].get("ip")
port = parsed["address"].get("port")
if ip and port:
master = "%s:%s" % (ip, port)
except Exception:
logger.exception("No JSON content, probably connecting "
"to older Mesos version.")
if master:
self.change_master(master)
def onNoMasterDetectedMessage(self):
self.change_master(None)
def gen_request(self):
data = json.dumps(
dict(
type='SUBSCRIBE',
subscribe=dict(framework_info=self.framework),
))
request = ('POST /api/v1/scheduler HTTP/1.1\r\nHost: %s\r\n'
'Content-Type: application/json\r\n'
'Accept: application/json\r\n'
'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % (
self.master, len(data), data)
return request.encode('utf-8')
def on_close(self):
if self._conn is not None:
self._conn.close()
self._conn = None
self.sched.disconnected(self)
def on_subscribed(self, info):
reregistered = (self.framework_id is not None)
self.framework_id = info['framework_id']['value']
hostname, port = self.master.split(':', 2)
port = int(port)
master_info = dict(
hostname=hostname,
port=port,
)
if self.version:
master_info['version'] = self.version
if reregistered:
self.sched.reregistered(self, master_info)
else:
framework_id = dict(value=self.framework_id)
self.sched.registered(self, framework_id, master_info)
def on_offers(self, event):
offers = event['offers']
self.sched.resourceOffers(self, offers)
def on_rescind(self, event):
offer_id = event['offer_id']
self.sched.offerRescinded(self, offer_id)
def on_update(self, event):
status = event['status']
self.sched.statusUpdate(self, status)
self.acknowledgeStatusUpdate(status)
def on_message(self, message):
executor_id = message['executor_id']
agent_id = message['agent_id']
data = message['data']
self.sched.frameworkMessage(self, executor_id, agent_id, data)
def on_failure(self, failure):
agent_id = failure['agent_id']
if 'executor_id' not in failure:
self.sched.slaveLost(self, agent_id)
else:
self.sched.executorLost(self, failure['executor_id'], agent_id,
failure['status'])
def on_error(self, event):
message = event['message']
self.sched.error(self, message)
def on_event(self, event):
if 'type' in event:
_type = event['type'].lower()
if _type == 'heartbeat':
return
if _type not in event:
logger.error('Missing `%s` in event %s' % (_type, event))
return
event = event[_type]
func_name = 'on_%s' % (_type, )
func = getattr(self, func_name, None)
if func is not None:
func(event)
else:
logger.error('Unknown type:%s, event:%s' % (_type, event))
else:
logger.error('Unknown event:%s' % (event, ))
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers):
user, pw = self.passwd.find_user_password(realm, req.get_full_url())
if pw is not None:
user_parts = user.split('\\', 1)
if len(user_parts) == 1:
UserName = user_parts[0]
DomainName = ''
type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied
else:
DomainName = user_parts[0].upper()
UserName = user_parts[1]
type1_flags = ntlm.NTLM_TYPE1_FLAGS
# ntlm secures a socket, so we must use the same socket for the complete handshake
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user, type1_flags).decode("utf-8")
if req.headers.get(self.auth_header, None) == auth:
return None
headers[self.auth_header] = auth
host = req.host
if not host:
raise urllib.request.URLError('no host given')
h = None
if req.get_full_url().startswith('https://'):
h = HTTPSConnection(host) # will parse host:port
else:
h = HTTPConnection(host) # will parse host:port
h.set_debuglevel(self._debuglevel)
# we must keep the connection because NTLM authenticates the connection, not single requests
headers["Connection"] = "Keep-Alive"
headers = dict((name.title(), val) for name, val in headers.items())
# For some reason, six doesn't do this translation correctly
# TODO rsanders low - find bug in six & fix it
try:
selector = req.selector
except AttributeError:
selector = req.get_selector()
h.request(req.get_method(), selector, req.data, headers)
r = h.getresponse()
r.begin()
r._safe_read(int(r.getheader('content-length')))
if r.getheader('set-cookie'):
# this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out)
headers['Cookie'] = r.getheader('set-cookie')
r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open)
auth_header_value = r.getheader(auth_header_field, None)
# some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge
# and another with the 'Negotiate' keyword - make sure we operate on the right one
m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value)
if m:
auth_header_value, = m.groups()
(ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value[5:])
auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, UserName, DomainName, pw,
NegotiateFlags).decode("utf-8")
headers[self.auth_header] = auth
headers["Connection"] = "Close"
headers = dict((name.title(), val) for name, val in headers.items())
try:
h.request(req.get_method(), selector, req.data, headers)
# none of the configured handlers are triggered, for example redirect-responses are not handled!
response = h.getresponse()
def notimplemented():
raise NotImplementedError
response.readline = notimplemented
infourl = urllib.response.addinfourl(response, response.msg, req.get_full_url())
infourl.code = response.status
infourl.msg = response.reason
return infourl
except socket.error as err:
raise urllib.error.URLError(err)
else:
return None
def __init__(self, uri, basepath=None):
self.basepath = basepath
self.mimetype = None
self.file = None
self.data = None
self.uri = None
self.local = None
self.tmp_file = None
uri = uri or str()
uri = uri.encode('utf-8')
log.debug("FileObject %r, Basepath: %r", uri, basepath)
# Data URI
if isinstance(uri, binary_type):
uri = uri.decode('utf-8')
if uri.startswith("data:"):
m = _rx_datauri.match(uri)
self.mimetype = m.group("mime")
self.data = base64.decodestring(m.group("data"))
else:
# Check if we have an external scheme
if basepath and not urlparse(uri).scheme:
urlParts = urlparse(basepath)
else:
urlParts = urlparse(uri)
log.debug("URLParts: %r", urlParts)
if urlParts.scheme == 'file':
if basepath and uri.startswith('/'):
uri = urljoin(basepath, uri[1:])
urlResponse = urlopen(uri)
self.mimetype = urlResponse.info().get("Content-Type",
'').split(";")[0]
self.uri = urlResponse.geturl()
self.file = urlResponse
# Drive letters have len==1 but we are looking
# for things like http:
elif urlParts.scheme in ('http', 'https'):
# External data
if basepath:
uri = urljoin(basepath, uri)
#path = urlsplit(url)[2]
#mimetype = getMimeType(path)
# Using HTTPLIB
parts = urlsplit(uri[uri.find("//"):])
server, path = parts.netloc, parts.path
if uri.startswith("https://"):
conn = HTTPSConnection(server)
else:
conn = HTTPConnection(server)
conn.request("GET", path)
r1 = conn.getresponse()
# log.debug("HTTP %r %r %r %r", server, path, uri, r1)
if (r1.status, r1.reason) == (200, "OK"):
self.mimetype = r1.getheader("Content-Type",
'').split(";")[0]
self.uri = uri
if r1.getheader("content-encoding") == "gzip":
import gzip
self.file = gzip.GzipFile(mode="rb",
fileobj=BytesIO(r1.read()))
else:
self.file = r1
else:
try:
urlResponse = urlopen(uri)
except HTTPError:
return
self.mimetype = urlResponse.info().get("Content-Type",
'').split(";")[0]
self.uri = urlResponse.geturl()
self.file = urlResponse
else:
# Local data
if basepath:
uri = os.path.normpath(os.path.join(basepath, uri))
if os.path.isfile(uri):
self.uri = uri
self.local = uri
self.setMimeTypeByName(uri)
self.file = open(uri, "rb")
def __init__(self, host, *fakedata):
HTTPConnection.__init__(self, host)
self.fakedata = list(fakedata)
class MesosSchedulerDriver(Process, SchedulerDriver):
_timeout = 10
def __init__(self,
sched,
framework,
master_uri,
use_addict=False,
implicit_acknowledgements=True,
principal=None,
secret=None,
failover=False,
timeout=DAY):
super(MesosSchedulerDriver, self).__init__(timeout=timeout)
self.sched = sched
self.master_uri = master_uri
self._framework = framework
self.detector = None
self._conn = None
self.version = None
self._failover = failover
self._dict_cls = Dict if use_addict else dict
self.implicit_acknowledgements = implicit_acknowledgements
if principal is not None and secret is not None:
self._basic_credential = 'Basic %s' % (b2a_base64(
('%s:%s' %
(principal, secret)).encode('ascii')).decode('ascii').strip())
else:
self._basic_credential = None
@property
def framework(self):
framework = dict(self._framework)
version = self.version and tuple(
int(n) for n in self.version.split('.'))
capabilities = [
c for c in framework.get('capabilities', [])
if c['type'] != 'GPU_RESOURCES'
]
if version and version >= (1, 0, 0):
capabilities.append(dict(type='GPU_RESOURCES'))
if capabilities:
framework['capabilities'] = capabilities
else:
framework.pop('capabilities', None)
if 'failover_timeout' not in framework:
framework['failover_timeout'] = 100
return framework
@property
def framework_id(self):
id = self._framework.get('id')
return id and id.get('value')
@framework_id.setter
def framework_id(self, id):
self._framework['id'] = dict(value=id)
def _get_version(self, master):
if master is not None:
conn = None
host, port = master.split(':', 2)
port = int(port)
try:
conn = HTTPConnection(host, port, timeout=self._timeout)
conn.request('GET', '/version')
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
return
return json.loads(resp.read().decode('utf-8'))['version']
except Exception:
logger.exception('Error')
pass
finally:
if conn:
conn.close()
def change_master(self, master):
self.version = self._get_version(master)
super(MesosSchedulerDriver, self).change_master(master)
self._close()
def start(self):
super(MesosSchedulerDriver, self).start()
uri = self.master_uri
if uri.startswith('zk://') or uri.startswith('zoo://'):
from .detector import MasterDetector
self.detector = MasterDetector(uri[uri.index('://') + 3:], self)
self.detector.start()
else:
if ':' not in uri:
uri += ':5050'
self.change_master(uri)
def stop(self, failover=False):
with self._lock:
self._failover = failover
detector = self.detector
self.detector = None
if detector:
detector.stop()
super(MesosSchedulerDriver, self).stop()
def _shutdown(self):
if not self._failover:
try:
self._teardown()
except Exception:
logger.exception('Failed to Teardown')
def _get_conn(self):
if not self.connected:
return None
if self._conn is not None:
return self._conn
host, port = self.master.split(':', 2)
port = int(port)
self._conn = HTTPConnection(host, port, timeout=self._timeout)
return self._conn
def _send(self, body, path='/api/v1/scheduler', method='POST', headers={}):
with self._lock:
conn = self._get_conn()
if conn is None:
raise RuntimeError('Not connected yet')
if body != '':
data = json.dumps(body).encode('utf-8')
headers['Content-Type'] = 'application/json'
else:
data = b''
stream_id = self.stream_id
if stream_id:
headers['Mesos-Stream-Id'] = stream_id
if self._basic_credential:
headers['Authorization'] = self._basic_credential
try:
conn.request(method, path, body=data, headers=headers)
resp = conn.getresponse()
except Exception:
self._close()
raise
if resp.status < 200 or resp.status >= 300:
raise RuntimeError('Failed to send request %s: %s\n%s' %
(resp.status, resp.read(), data))
result = resp.read()
if not result:
return {}
try:
return json.loads(result.decode('utf-8'))
except Exception:
return {}
def _teardown(self):
if self.connected:
framework_id = self.framework_id
if framework_id:
self._send(
dict(
type='TEARDOWN',
framework_id=dict(value=framework_id, ),
))
self._framework.pop('id', None)
def acceptOffers(self, offer_ids, operations, filters=None):
if not operations:
return self.declineOffer(offer_ids, filters=filters)
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
accept = dict(
offer_ids=[offer_ids]
if isinstance(offer_ids, dict) else offer_ids,
operations=operations,
)
if filters is not None:
accept['filters'] = filters
body = dict(
type='ACCEPT',
framework_id=dict(value=framework_id, ),
accept=accept,
)
self._send(body)
def acceptInverseOffers(self, offer_ids, filters=None):
framework_id = self.framework_id
assert framework_id
accept_inverse_offers = dict(
inverse_offer_ids=[offer_ids] if isinstance(offer_ids, dict
) else offer_ids)
if filters is not None:
accept_inverse_offers['filters'] = filters
body = dict(
type='ACCEPT_INVERSE_OFFERS',
framework_id=dict(value=framework_id, ),
accept_inverse_offers=accept_inverse_offers,
)
self._send(body)
def launchTasks(self, offer_ids, tasks, filters=None):
if not tasks:
return self.declineOffer(offer_ids, filters=filters)
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
operations = [dict(
type='LAUNCH',
launch=dict(task_infos=tasks),
)]
self.acceptOffers(offer_ids, operations, filters=filters)
def declineOffer(self, offer_ids, filters=None):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
decline = dict(offer_ids=[offer_ids] if isinstance(offer_ids, dict
) else offer_ids)
if filters is not None:
decline['filters'] = filters
body = dict(
type='DECLINE',
framework_id=dict(value=framework_id, ),
decline=decline,
)
self._send(body)
def declineInverseOffer(self, offer_ids, filters=None):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
decline_inverse_offers = dict(
inverse_offer_ids=[offer_ids] if isinstance(offer_ids, dict
) else offer_ids)
if filters is not None:
decline_inverse_offers['filters'] = filters
body = dict(
type='DECLINE_INVERSE_OFFERS',
framework_id=dict(value=framework_id, ),
decline_inverse_offers=decline_inverse_offers,
)
self._send(body)
def reviveOffers(self):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='REVIVE',
framework_id=dict(value=framework_id, ),
)
self._send(body)
def suppressOffers(self):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='SUPPRESS',
framework_id=dict(value=framework_id, ),
)
self._send(body)
def killTask(self, task_id):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='KILL',
framework_id=dict(value=framework_id, ),
kill=dict(task_id=task_id, ),
)
self._send(body)
def acknowledgeStatusUpdate(self, status):
if self.connected and 'uuid' in status:
framework_id = self.framework_id
assert framework_id
acknowledge = dict()
acknowledge['agent_id'] = status['agent_id']
acknowledge['task_id'] = status['task_id']
acknowledge['uuid'] = status['uuid']
body = dict(
type='ACKNOWLEDGE',
framework_id=dict(value=framework_id, ),
acknowledge=acknowledge,
)
self._send(body)
def acknowledgeOperationStatusUpdate(self, status):
if self.connected and 'uuid' in status and 'operation_id' in status:
framework_id = self.framework_id
assert framework_id
body = dict(
type='ACKNOWLEDGE_OPERATION_STATUS',
framework_id=dict(value=framework_id, ),
acknowledge_operation_status=status,
)
self._send(body)
def reconcileTasks(self, tasks):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='RECONCILE',
framework_id=dict(value=framework_id, ),
reconcile=dict(
tasks=[dict(task_id=task['task_id']) for task in tasks], ),
)
self._send(body)
def reconcileOperations(self, operations_):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
operations = []
for op_ in operations_:
op = dict(operation_id=op_['operation_id'])
if 'agent_id' in op_:
op['agent_id'] = op_['agent_id']
if 'resource_provider_id' in op_:
op['resource_provider_id'] = op_['resource_provider_id']
operations.append(op)
body = dict(
type='RECONCILE_OPERATIONS',
framework_id=dict(value=framework_id, ),
reconcile_operations=dict(operations=operations, ),
)
self._send(body)
def sendFrameworkMessage(self, executor_id, agent_id, data):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
message = dict(
agent_id=agent_id,
executor_id=executor_id,
data=data,
)
body = dict(
type='MESSAGE',
framework_id=dict(value=framework_id, ),
message=message,
)
self._send(body)
def requestResources(self, requests):
if not self.connected:
return
framework_id = self.framework_id
assert framework_id
body = dict(
type='REQUEST',
framework_id=dict(value=framework_id, ),
request=dict(requests=requests, ),
)
self._send(body)
def onNewMasterDetectedMessage(self, data):
master = None
try:
if isinstance(data, six.binary_type):
data = data.decode('utf-8')
parsed = json.loads(data)
if parsed and "address" in parsed:
ip = parsed["address"].get("ip")
port = parsed["address"].get("port")
if ip and port:
master = "%s:%s" % (ip, port)
except Exception:
logger.exception("No JSON content, probably connecting "
"to older Mesos version.")
if master:
self.change_master(master)
def onNoMasterDetectedMessage(self):
self.change_master(None)
def gen_request(self):
request = dict(
type='SUBSCRIBE',
subscribe=dict(framework_info=self.framework),
)
if 'id' in self._framework:
request['framework_id'] = self._framework['id']
data = json.dumps(request)
_authorization = ''
if self._basic_credential is not None:
_authorization = 'Authorization: %s\r\n' % (
self._basic_credential, )
request = ('POST /api/v1/scheduler HTTP/1.1\r\nHost: %s\r\n'
'Content-Type: application/json\r\n'
'Accept: application/json\r\n%s'
'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % (
self.master, _authorization, len(data), data)
return request.encode('utf-8')
def _close(self):
if self._conn is not None:
self._conn.close()
self._conn = None
def on_close(self):
self._close()
self.sched.disconnected(self)
def on_subscribed(self, info):
reregistered = (self.framework_id is not None)
self.framework_id = info['framework_id']['value']
hostname, port = self.master.split(':', 2)
port = int(port)
master_info = dict(
hostname=hostname,
port=port,
)
if self.version:
master_info['version'] = self.version
elif 'master_info' in info and 'version' in info['master_info']:
master_info['version'] = info['master_info']['version']
if reregistered:
self.sched.reregistered(self, self._dict_cls(master_info))
else:
framework_id = dict(value=self.framework_id)
self.sched.registered(self, self._dict_cls(framework_id),
self._dict_cls(master_info))
def on_offers(self, event):
offers = event.get('offers', [])
if offers:
self.sched.resourceOffers(
self, [self._dict_cls(offer) for offer in offers])
version = self.version and tuple(
int(n) for n in self.version.split('.'))
if not (version and version >= (1, 0, 0)):
self.on_inverse_offers(event)
def on_inverse_offers(self, event):
inverse_offers = event.get('inverse_offers', [])
if inverse_offers:
self.sched.inverseOffers(
self, [self._dict_cls(offer) for offer in inverse_offers])
def on_rescind(self, event):
offer_id = event['offer_id']
self.sched.offerRescinded(self, self._dict_cls(offer_id))
def on_rescind_inverse_offer(self, event):
inverse_offer_id = event['inverse_offer_id']
self.sched.inverseOfferRescinded(self,
self._dict_cls(inverse_offer_id))
def on_update(self, event):
status = event['status']
self.sched.statusUpdate(self, self._dict_cls(status))
if self.implicit_acknowledgements:
self.acknowledgeStatusUpdate(status)
def on_update_operation_status(self, event):
status = event['status']
self.sched.operationStatusUpdate(self, self._dict_cls(status))
if self.implicit_acknowledgements:
self.acknowledgeOperationStatusUpdate(status)
def on_message(self, message):
executor_id = message['executor_id']
agent_id = message['agent_id']
data = message['data']
self.sched.frameworkMessage(self, self._dict_cls(executor_id),
self._dict_cls(agent_id), data)
def on_failure(self, failure):
agent_id = failure['agent_id']
if 'executor_id' not in failure:
self.sched.slaveLost(self, self._dict_cls(agent_id))
else:
self.sched.executorLost(self,
self._dict_cls(failure['executor_id']),
self._dict_cls(agent_id),
failure['status'])
def on_error(self, event):
message = event['message']
self.sched.error(self, message)
def on_heartbeat(self):
self.sched.processHeartBeat()
def on_event(self, event):
if 'type' in event:
_type = event['type'].lower()
if _type == 'heartbeat':
self.on_heartbeat()
return
if _type not in event:
logger.error('Missing `%s` in event %s' % (_type, event))
return
event = event[_type]
func_name = 'on_%s' % (_type, )
func = getattr(self, func_name, None)
if func is not None:
func(event)
else:
logger.error('Unknown type:%s, event:%s' % (_type, event))
else:
logger.error('Unknown event:%s' % (event, ))
class GoogleSuggestions():
def __init__(self):
self.hl = "en"
self.conn = None
def prepareQuery(self):
#GET /complete/search?output=toolbar&client=youtube-psuggest&xml=true&ds=yt&hl=en&jsonp=self.gotSuggestions&q=s
#self.prepQuerry = "/complete/search?output=toolbar&client=youtube&xml=true&ds=yt&"
self.prepQuerry = "/complete/search?output=chrome&client=chrome&"
if self.hl is not None:
self.prepQuerry = self.prepQuerry + "hl=" + self.hl + "&"
self.prepQuerry = self.prepQuerry + "jsonp=self.gotSuggestions&q="
print("[MyTube - GoogleSuggestions] prepareQuery:", self.prepQuerry)
def getSuggestions(self, queryString):
self.prepareQuery()
if queryString != "":
query = self.prepQuerry + quote(queryString)
self.conn = HTTPConnection("google.com")
try:
self.conn = HTTPConnection("google.com")
self.conn.request("GET", query, "", {"Accept-Encoding": "UTF-8"})
except (CannotSendRequest, gaierror, error):
self.conn.close()
print("[MyTube - GoogleSuggestions] Can not send request for suggestions")
return None
else:
try:
response = self.conn.getresponse()
except BadStatusLine:
self.conn.close()
print("[MyTube - GoogleSuggestions] Can not get a response from google")
return None
else:
if response.status == 200:
data = response.read()
header = response.getheader("Content-Type", "text/xml; charset=ISO-8859-1")
charset = "ISO-8859-1"
try:
charset = header.split(";")[1].split("=")[1]
print("[MyTube - GoogleSuggestions] Got charset %s" % charset)
except:
print("[MyTube - GoogleSuggestions] No charset in Header, falling back to %s" % charset)
data = data.decode(charset).encode("utf-8")
self.conn.close()
return data
else:
self.conn.close()
return None
else:
return None
def make_connection(self, server):
host, port = server.server_address
connection = HTTPConnection(host, port)
self.addCleanup(connection.close)
return connection
def test_fails(self, HTTPConnection):
# coverage
conn = HTTPConnection()
conn.request.side_effect = socket.error
rv = self.call_it('127.0.0.1', 8080, timeout=0.01, retries=2)
self.assertFalse(rv)
def make_connection(self):
if self.scheme == 'https':
return HTTPSConnection('%s:%s' % (self.interface(), self.PORT))
else:
return HTTPConnection('%s:%s' % (self.interface(), self.PORT))
class Client:
"""
Crossbar.io HTTP bridge client.
"""
def __init__(self, url, key = None, secret = None, timeout = 5, context = None):
"""
Create a new Crossbar.io push client.
The only mandatory argument is the Push service endpoint of the Crossbar.io
instance to push to.
For signed pushes, provide authentication key and secret. If those are not
given, unsigned pushes are performed.
:param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push).
:type url: str
:param key: Optional key to use for signing requests.
:type key: str
:param secret: When using signed request, the secret corresponding to key.
:type secret: str
:param timeout: Timeout for requests.
:type timeout: int
:param context: If the HTTP bridge is running on HTTPS (that is securely over TLS),
then the context provides the SSL settings the client should use (e.g. the
certificate chain against which to verify the server certificate). This parameter
is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently
ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext
:type context: obj or None
"""
if six.PY2:
if type(url) == str:
url = six.u(url)
if type(key) == str:
key = six.u(key)
if type(secret) == str:
secret = six.u(secret)
assert(type(url) == six.text_type)
assert((key and secret) or (not key and not secret))
assert(key is None or type(key) == six.text_type)
assert(secret is None or type(secret) == six.text_type)
assert(type(timeout) == int)
if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT:
assert(context is None or isinstance(context, ssl.SSLContext))
self._seq = 1
self._key = key
self._secret = secret
self._endpoint = _parse_url(url)
self._endpoint['headers'] = {
"Content-type": "application/json",
"User-agent": "crossbarconnect-python"
}
if self._endpoint['secure']:
if not _HAS_SSL:
raise Exception("Bridge URL is using HTTPS, but Python SSL module is missing")
if _HAS_SSL_CLIENT_CONTEXT:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'], timeout = timeout, context = context)
else:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'], timeout = timeout)
else:
self._connection = HTTPConnection(self._endpoint['host'],
self._endpoint['port'], timeout = timeout)
def publish(self, topic, *args, **kwargs):
"""
Publish an event to subscribers on specified topic via Crossbar.io HTTP bridge.
The event payload (positional and keyword) can be of any type that can be
serialized to JSON.
If `kwargs` contains an `options` attribute, this is expected to
be a dictionary with the following possible parameters:
* `exclude`: A list of WAMP session IDs to exclude from receivers.
* `eligible`: A list of WAMP session IDs eligible as receivers.
:param topic: Topic to push to.
:type topic: str
:param args: Arbitrary application payload for the event (positional arguments).
:type args: list
:param kwargs: Arbitrary application payload for the event (keyword arguments).
:type kwargs: dict
:returns int -- The event publication ID assigned by the broker.
"""
if six.PY2 and type(topic) == str:
topic = six.u(topic)
assert(type(topic) == six.text_type)
## this will get filled and later serialized into HTTP/POST body
##
event = {
'topic': topic
}
if 'options' in kwargs:
event['options'] = kwargs.pop('options')
assert(type(event['options']) == dict)
if args:
event['args'] = args
if kwargs:
event['kwargs'] = kwargs
try:
body = json.dumps(event, separators = (',',':'))
if six.PY3:
body = body.encode('utf8')
except Exception as e:
raise Exception("invalid event payload - not JSON serializable: {0}".format(e))
params = {
'timestamp': _utcnow(),
'seq': self._seq,
}
if self._key:
## if the request is to be signed, create extra fields and signature
params['key'] = self._key
params['nonce'] = random.randint(0, 9007199254740992)
# HMAC[SHA256]_{secret} (key | timestamp | seq | nonce | body) => signature
hm = hmac.new(self._secret.encode('utf8'), None, hashlib.sha256)
hm.update(params['key'].encode('utf8'))
hm.update(params['timestamp'].encode('utf8'))
hm.update(u"{0}".format(params['seq']).encode('utf8'))
hm.update(u"{0}".format(params['nonce']).encode('utf8'))
hm.update(body)
signature = base64.urlsafe_b64encode(hm.digest())
params['signature'] = signature
self._seq += 1
path = "{0}?{1}".format(parse.quote(self._endpoint['path']), parse.urlencode(params))
## now issue the HTTP/POST
##
self._connection.request('POST', path, body, self._endpoint['headers'])
response = self._connection.getresponse()
response_body = response.read().decode()
if response.status != 202:
raise Exception("publication request failed {0} [{1}] - {2}".format(response.status, response.reason, response_body))
try:
res = json.loads(response_body)
except Exception as e:
raise Exception("publication request bogus result - {0}".format(e))
return res['id']
class Client:
"""
Crossbar.io HTTP bridge client.
"""
def __init__(self, url, key=None, secret=None, timeout=5, context=None):
"""
Create a new Crossbar.io push client.
The only mandatory argument is the Push service endpoint of the Crossbar.io
instance to push to.
For signed pushes, provide authentication key and secret. If those are not
given, unsigned pushes are performed.
:param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push).
:type url: str
:param key: Optional key to use for signing requests.
:type key: str
:param secret: When using signed request, the secret corresponding to key.
:type secret: str
:param timeout: Timeout for requests.
:type timeout: int
:param context: If the HTTP bridge is running on HTTPS (that is securely over TLS),
then the context provides the SSL settings the client should use (e.g. the
certificate chain against which to verify the server certificate). This parameter
is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently
ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext
:type context: obj or None
"""
if six.PY2:
if type(url) == str:
url = six.u(url)
if type(key) == str:
key = six.u(key)
if type(secret) == str:
secret = six.u(secret)
assert (type(url) == six.text_type)
assert ((key and secret) or (not key and not secret))
assert (key is None or type(key) == six.text_type)
assert (secret is None or type(secret) == six.text_type)
assert (type(timeout) == int)
if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT:
assert (context is None or isinstance(context, ssl.SSLContext))
self._seq = 1
self._key = key
self._secret = secret
self._endpoint = _parse_url(url)
self._endpoint['headers'] = {
"Content-type": "application/json",
"User-agent": "crossbarconnect-python"
}
if self._endpoint['secure']:
if not _HAS_SSL:
raise Exception(
"Bridge URL is using HTTPS, but Python SSL module is missing"
)
if _HAS_SSL_CLIENT_CONTEXT:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout,
context=context)
else:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout)
else:
self._connection = HTTPConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout)
def publish(self, topic, *args, **kwargs):
"""
Publish an event to subscribers on specified topic via Crossbar.io HTTP bridge.
The event payload (positional and keyword) can be of any type that can be
serialized to JSON.
If `kwargs` contains an `options` attribute, this is expected to
be a dictionary with the following possible parameters:
* `exclude`: A list of WAMP session IDs to exclude from receivers.
* `eligible`: A list of WAMP session IDs eligible as receivers.
:param topic: Topic to push to.
:type topic: str
:param args: Arbitrary application payload for the event (positional arguments).
:type args: list
:param kwargs: Arbitrary application payload for the event (keyword arguments).
:type kwargs: dict
:returns int -- The event publication ID assigned by the broker.
"""
if six.PY2 and type(topic) == str:
topic = six.u(topic)
assert (type(topic) == six.text_type)
## this will get filled and later serialized into HTTP/POST body
##
event = {'topic': topic}
if 'options' in kwargs:
event['options'] = kwargs.pop('options')
assert (type(event['options']) == dict)
if args:
event['args'] = args
if kwargs:
event['kwargs'] = kwargs
try:
body = json.dumps(event, separators=(',', ':'))
if six.PY3:
body = body.encode('utf8')
except Exception as e:
raise Exception(
"invalid event payload - not JSON serializable: {0}".format(e))
params = {
'timestamp': _utcnow(),
'seq': self._seq,
}
if self._key:
## if the request is to be signed, create extra fields and signature
params['key'] = self._key
params['nonce'] = random.randint(0, 9007199254740992)
# HMAC[SHA256]_{secret} (key | timestamp | seq | nonce | body) => signature
hm = hmac.new(self._secret.encode('utf8'), None, hashlib.sha256)
hm.update(params['key'].encode('utf8'))
hm.update(params['timestamp'].encode('utf8'))
hm.update(u"{0}".format(params['seq']).encode('utf8'))
hm.update(u"{0}".format(params['nonce']).encode('utf8'))
hm.update(body)
signature = base64.urlsafe_b64encode(hm.digest())
params['signature'] = signature
self._seq += 1
path = "{0}?{1}".format(parse.quote(self._endpoint['path']),
parse.urlencode(params))
## now issue the HTTP/POST
##
self._connection.request('POST', path, body, self._endpoint['headers'])
response = self._connection.getresponse()
response_body = response.read()
if response.status != 202:
raise Exception(
"publication request failed {0} [{1}] - {2}".format(
response.status, response.reason, response_body))
try:
res = json.loads(response_body)
except Exception as e:
raise Exception("publication request bogus result - {0}".format(e))
return res['id']
def __init__(self, path):
HTTPConnection.__init__(self, 'localhost')
self.path = path
def __init__(self, *args, **kwargs):
HTTPConnection.__init__(self, *args, **kwargs)
self._canceled = False
self.deadline = 0
def __init__(self, url, key=None, secret=None, timeout=5, context=None):
"""
Create a new Crossbar.io push client.
The only mandatory argument is the Push service endpoint of the Crossbar.io
instance to push to.
For signed pushes, provide authentication key and secret. If those are not
given, unsigned pushes are performed.
:param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push).
:type url: str
:param key: Optional key to use for signing requests.
:type key: str
:param secret: When using signed request, the secret corresponding to key.
:type secret: str
:param timeout: Timeout for requests.
:type timeout: int
:param context: If the HTTP bridge is running on HTTPS (that is securely over TLS),
then the context provides the SSL settings the client should use (e.g. the
certificate chain against which to verify the server certificate). This parameter
is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently
ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext
:type context: obj or None
"""
if six.PY2:
if type(url) == str:
url = six.u(url)
if type(key) == str:
key = six.u(key)
if type(secret) == str:
secret = six.u(secret)
assert (type(url) == six.text_type)
assert ((key and secret) or (not key and not secret))
assert (key is None or type(key) == six.text_type)
assert (secret is None or type(secret) == six.text_type)
assert (type(timeout) == int)
if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT:
assert (context is None or isinstance(context, ssl.SSLContext))
self._seq = 1
self._key = key
self._secret = secret
self._endpoint = _parse_url(url)
self._endpoint['headers'] = {
"Content-type": "application/json",
"User-agent": "crossbarconnect-python"
}
if self._endpoint['secure']:
if not _HAS_SSL:
raise Exception(
"Bridge URL is using HTTPS, but Python SSL module is missing"
)
if _HAS_SSL_CLIENT_CONTEXT:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout,
context=context)
else:
self._connection = HTTPSConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout)
else:
self._connection = HTTPConnection(self._endpoint['host'],
self._endpoint['port'],
timeout=timeout)
