def report_sauce_status(self, name, status, tags=[], remote_url=''): """Report test status and tags to SauceLabs """ job_id = BuiltIn().get_library_instance( 'Selenium2Library')._current_browser().session_id if USERNAME_ACCESS_KEY.match(remote_url): username, access_key =\ USERNAME_ACCESS_KEY.findall(remote_url)[0][1:] else: username = os.environ.get('SAUCE_USERNAME') access_key = os.environ.get('SAUCE_ACCESS_KEY') if not job_id: return u"No Sauce job id found. Skipping..." elif not username or not access_key: return u"No Sauce environment variables found. Skipping..." token = base64.encodestring('%s:%s' % (username, access_key))[:-1] body = json.dumps({'name': name, 'passed': status == 'PASS', 'tags': tags}) connection = HTTPConnection('saucelabs.com') connection.request('PUT', '/rest/v1/%s/jobs/%s' % ( username, job_id), body, headers={'Authorization': 'Basic %s' % token} ) return connection.getresponse().status
def __init__(self, url, key = None, secret = None, timeout = 5, context = None): """ Create a new Crossbar.io push client. The only mandatory argument is the Push service endpoint of the Crossbar.io instance to push to. For signed pushes, provide authentication key and secret. If those are not given, unsigned pushes are performed. :param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push). :type url: str :param key: Optional key to use for signing requests. :type key: str :param secret: When using signed request, the secret corresponding to key. :type secret: str :param timeout: Timeout for requests. :type timeout: int :param context: If the HTTP bridge is running on HTTPS (that is securely over TLS), then the context provides the SSL settings the client should use (e.g. the certificate chain against which to verify the server certificate). This parameter is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext :type context: obj or None """ if six.PY2: if type(url) == str: url = six.u(url) if type(key) == str: key = six.u(key) if type(secret) == str: secret = six.u(secret) assert(type(url) == six.text_type) assert((key and secret) or (not key and not secret)) assert(key is None or type(key) == six.text_type) assert(secret is None or type(secret) == six.text_type) assert(type(timeout) == int) if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT: assert(context is None or isinstance(context, ssl.SSLContext)) self._seq = 1 self._key = key self._secret = secret self._endpoint = _parse_url(url) self._endpoint['headers'] = { "Content-type": "application/json", "User-agent": "crossbarconnect-python" } if self._endpoint['secure']: if not _HAS_SSL: raise Exception("Bridge URL is using HTTPS, but Python SSL module is missing") if _HAS_SSL_CLIENT_CONTEXT: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout = timeout, context = context) else: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout = timeout) else: self._connection = HTTPConnection(self._endpoint['host'], self._endpoint['port'], timeout = timeout)
def connect(self): """ Override the connect() function to intercept calls to certain host/ports. If no app at host/port has been registered for interception then a normal HTTPConnection is made. """ if debuglevel: sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,)) try: (app, script_name) = self.get_app(self.host, self.port) if app: if debuglevel: sys.stderr.write('INTERCEPTING call to %s:%s\n' % (self.host, self.port,)) self.sock = wsgi_fake_socket(app, self.host, self.port, script_name) else: HTTPConnection.connect(self) except Exception: if debuglevel: # intercept & print out tracebacks traceback.print_exc() raise
class WebcamStreamBase(VideoStream): request_headers = { 'Accept': '*/*', } def __init__(self, url, max_rate=3.0, rate_bucket_size=None, socket_timeout=10, user_agent=DEFAULT_USER_AGENT): self.url = url netloc, self.path = _parse_url(url) self.conn = HTTPConnection(netloc, timeout=socket_timeout) self.request_headers = self.request_headers.copy() self.request_headers['User-Agent'] = user_agent self.stream = None self.rate_limiter = BucketRateLimiter(max_rate, rate_bucket_size) self.open_rate_limiter = BackoffRateLimiter(socket_timeout) def __repr__(self): return u"<%s at 0x%x: %s>" % ( self.__class__.__name__, id(self), self.url) @classmethod def from_settings(cls, settings, prefix='webcam.', **defaults): config = config_from_settings(settings, prefix=prefix, subprefix=cls.settings_subprefix, **defaults) return cls(**config) def close(self): if self.stream: self.stream = None if self.conn: self.conn.close() self.conn = None @property def closed(self): return not self.conn def next(self): # FIXME: check closed more often? if self.closed: raise StopIteration() next(self.rate_limiter) try: if self.stream is None: next(self.open_rate_limiter) self.stream = self._open_stream() frame = next(self.stream) self.open_rate_limiter.reset() return frame except Exception as ex: self.stream = None log.warn("Streaming failed: %s", text_type(ex) or repr(ex)) self.conn.close() return None
def _check_storage(ipport, path): conn = HTTPConnection(*ipport) conn.request('GET', path) resp = conn.getresponse() # 404 because it's a nonsense path (and mount_check is false) # 507 in case the test target is a VM using mount_check if resp.status not in (404, 507): raise Exception( 'Unexpected status %s' % resp.status) return resp
def set_test_status(self, passed=True): connection = HTTPConnection("saucelabs.com") connection.request( 'PUT', '/rest/v1/{0}/jobs/{1}'.format( self.username, self.driver.session_id ), json.dumps({"passed": passed}), headers={"Authorization": "Basic {0}".format(self.sauce_auth)} ) result = connection.getresponse() return result.status == 200
def open_http_connection(self, code, query=None, method='GET'): """ Send a request to non-sandboxed Splash, return an HTTPConnection. create_file Lua function is pre-loaded. XXX: why can't we use requests or urllib, why don't they close a connection after a timeout error? """ q = {"lua_source": self.CREATE_FILE + "\n" + code} q.update(query or {}) conn = HTTPConnection('localhost', self.splash_unrestricted.portnum) conn.request(method, "/execute/?" + six.moves.urllib.parse.urlencode(q)) return conn
def wait_for_server_to_hangup(ipport): try_until = time() + 30 while True: try: conn = HTTPConnection(*ipport) conn.request('GET', '/') conn.getresponse() except Exception: break if time() > try_until: raise Exception( 'Still answering on %s:%s after 30 seconds' % ipport) sleep(0.1)
def __init__(self, host, port=None, key_file=None, cert_file=None, ca_certs=None, strict=None, **kwargs): if six.PY2: HTTPConnection.__init__(self, host=host, port=port, strict=strict, **kwargs) else: # python3's HTTPConnection dropped the strict argument. HTTPConnection.__init__(self, host=host, port=port, **kwargs) self.key_file = key_file self.cert_file = cert_file self.ca_certs = ca_certs if self.ca_certs: self.cert_reqs = ssl.CERT_REQUIRED else: self.cert_reqs = ssl.CERT_NONE
def is_alive(self): """Test that the connection is still alive. Because the remote communication happens over HTTP we need to make an explicit request to the remote. It is allowed for WebDriver spec tests to not have a WebDriver session, since this may be what is tested. An HTTP request to an invalid path that results in a 404 is proof enough to us that the server is alive and kicking. """ conn = HTTPConnection(self.server.host, self.server.port) conn.request("HEAD", self.server.base_path + "invalid") res = conn.getresponse() return res.status == 404
def status(self): succubus_status = super(AlppacaDaemon, self).status() if succubus_status != 0: print("succubus_status is {0}".format(str(succubus_status))) return succubus_status conn = HTTPConnection('169.254.169.254', timeout=0.1) try: conn.request("GET", "/") conn.getresponse() except Exception as e: print("Error: alppaca is not reachable via IP 169.254.169.254. {0}".format(e)) return 3 else: return 0
def kill_server(ipport, ipport2server, pids): server, number = get_server_number(ipport, ipport2server) err = Manager([server]).kill(number=number) if err: raise Exception("unable to kill %s" % (server if not number else "%s%s" % (server, number))) try_until = time() + 30 while True: try: conn = HTTPConnection(*ipport) conn.request("GET", "/") conn.getresponse() except Exception as err: break if time() > try_until: raise Exception("Still answering on %s:%s after 30 seconds" % ipport) sleep(0.1)
def _get_conn(self): if self._conn is not None: return self._conn host, port = self._daemon.split(':', 2) port = int(port) self._conn = HTTPConnection(host, port, timeout=self._timeout) return self._conn
def send_email(request): try: recipients = request.GET['to'].split(',') url = request.GET['url'] proto, server, path, query, frag = urlsplit(url) if query: path += '?' + query conn = HTTPConnection(server) conn.request('GET',path) try: # Python 2.7+, use buffering of HTTP responses resp = conn.getresponse(buffering=True) except TypeError: # Python 2.6 and older resp = conn.getresponse() assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason) rawData = resp.read() conn.close() message = MIMEMultipart() message['Subject'] = "Graphite Image" message['To'] = ', '.join(recipients) message['From'] = 'composer@%s' % gethostname() text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(),url) ) image = MIMEImage( rawData ) image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png")) message.attach(text) message.attach(image) s = SMTP(settings.SMTP_SERVER) s.sendmail('composer@%s' % gethostname(),recipients,message.as_string()) s.quit() return HttpResponse( "OK" ) except: return HttpResponse( format_exc() )
def check_server(ipport, ipport2server, pids, timeout=CHECK_SERVER_TIMEOUT): server = ipport2server[ipport] if server[:-1] in ('account', 'container', 'object'): if int(server[-1]) > 4: return None path = '/connect/1/2' if server[:-1] == 'container': path += '/3' elif server[:-1] == 'object': path += '/3/4' try_until = time() + timeout while True: try: conn = HTTPConnection(*ipport) conn.request('GET', path) resp = conn.getresponse() # 404 because it's a nonsense path (and mount_check is false) # 507 in case the test target is a VM using mount_check if resp.status not in (404, 507): raise Exception( 'Unexpected status %s' % resp.status) break except Exception as err: if time() > try_until: print(err) print('Giving up on %s:%s after %s seconds.' % ( server, ipport, timeout)) raise err sleep(0.1) else: try_until = time() + timeout while True: try: url, token = get_auth('http://%s:%d/auth/v1.0' % ipport, 'test:tester', 'testing') account = url.split('/')[-1] head_account(url, token) return url, token, account except Exception as err: if time() > try_until: print(err) print('Giving up on proxy:8080 after 30 seconds.') raise err sleep(0.1) return None
def predict(self, examples, **kwargs): """Run prediction over HTTP/REST. :param examples: The input examples :return: The outcomes """ verify_example(examples, self.input_keys) request = self.create_request(examples) conn = HTTPConnection(self.hostname, self.port) conn.request('POST', self.path, json.dumps(request), self.headers) response = conn.getresponse().read() outcomes_list = json.loads(response) if "error" in outcomes_list: raise ValueError("remote server returns error: {0}".format(outcomes_list["error"])) outcomes_list = outcomes_list["outputs"] outcomes_list = self.deserialize_response(examples, outcomes_list) return outcomes_list
def check_server(ipport, ipport2server, pids, timeout=CHECK_SERVER_TIMEOUT): server = ipport2server[ipport] if server[:-1] in ("account", "container", "object"): if int(server[-1]) > 4: return None path = "/connect/1/2" if server[:-1] == "container": path += "/3" elif server[:-1] == "object": path += "/3/4" try_until = time() + timeout while True: try: conn = HTTPConnection(*ipport) conn.request("GET", path) resp = conn.getresponse() # 404 because it's a nonsense path (and mount_check is false) # 507 in case the test target is a VM using mount_check if resp.status not in (404, 507): raise Exception("Unexpected status %s" % resp.status) break except Exception as err: if time() > try_until: print(err) print("Giving up on %s:%s after %s seconds." % (server, ipport, timeout)) raise err sleep(0.1) else: try_until = time() + timeout while True: try: url, token = get_auth("http://%s:%d/auth/v1.0" % ipport, "test:tester", "testing") account = url.split("/")[-1] head_account(url, token) return url, token, account except Exception as err: if time() > try_until: print(err) print("Giving up on proxy:8080 after 30 seconds.") raise err sleep(0.1) return None
def _get_conn(self): if not self.connected: return None if self._conn is not None: return self._conn host, port = self.master.split(':', 2) port = int(port) self._conn = HTTPConnection(host, port, timeout=self._timeout) return self._conn
def compute_engine_id(): """Gets the Compute Engine project ID if it can be inferred. Uses 169.254.169.254 for the metadata server to avoid request latency from DNS lookup. See https://cloud.google.com/compute/docs/metadata#metadataserver for information about this IP address. (This IP is also used for Amazon EC2 instances, so the metadata flavor is crucial.) See https://github.com/google/oauth2client/issues/93 for context about DNS latency. :rtype: string or ``NoneType`` :returns: Compute Engine project ID if the metadata service is available, else ``None``. """ host = '169.254.169.254' uri_path = '/computeMetadata/v1/project/project-id' headers = {'Metadata-Flavor': 'Google'} connection = HTTPConnection(host, timeout=0.1) try: connection.request('GET', uri_path, headers=headers) response = connection.getresponse() if response.status == 200: return response.read() except socket.error: # socket.timeout or socket.error(64, 'Host is down') pass finally: connection.close()
def __init__(self, url, max_rate=3.0, rate_bucket_size=None, socket_timeout=10, user_agent=DEFAULT_USER_AGENT): self.url = url netloc, self.path = _parse_url(url) self.conn = HTTPConnection(netloc, timeout=socket_timeout) self.request_headers = self.request_headers.copy() self.request_headers['User-Agent'] = user_agent self.stream = None self.rate_limiter = BucketRateLimiter(max_rate, rate_bucket_size) self.open_rate_limiter = BackoffRateLimiter(socket_timeout)
def test_garbage_in(self): # Connect without SSL regardless of server.scheme c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) c._output(b'gjkgjklsgjklsgjkljklsg') c._send_output() response = c.response_class(c.sock, method='GET') try: response.begin() self.assertEqual(response.status, 400) self.assertEqual(response.fp.read(22), b'Malformed Request-Line') c.close() except socket.error: e = sys.exc_info()[1] # "Connection reset by peer" is also acceptable. if e.errno != errno.ECONNRESET: raise
def export_to_myexp(self, trans, id, myexp_username, myexp_password): """ Exports a workflow to myExperiment website. """ trans.workflow_building_mode = workflow_building_modes.ENABLED stored = self.get_stored_workflow(trans, id, check_ownership=False, check_accessible=True) # Convert workflow to dict. workflow_dict = self._workflow_to_dict(trans, stored) # # Create and submit workflow myExperiment request. # # Create workflow content JSON. workflow_content = json.dumps(workflow_dict, indent=4, sort_keys=True) # Create myExperiment request. request_raw = trans.fill_template( "workflow/myexp_export.mako", workflow_name=workflow_dict['name'], workflow_description=workflow_dict['annotation'], workflow_content=workflow_content, workflow_svg=self._workflow_to_svg_canvas(trans, stored).tostring() ) # strip() b/c myExperiment XML parser doesn't allow white space before XML; utf-8 handles unicode characters. request = unicodify(request_raw.strip(), 'utf-8') # Do request and get result. auth_header = base64.b64encode('%s:%s' % (myexp_username, myexp_password)) headers = {"Content-type": "text/xml", "Accept": "text/xml", "Authorization": "Basic %s" % auth_header} myexp_url = trans.app.config.get("myexperiment_url", self.__myexp_url) conn = HTTPConnection(myexp_url) # NOTE: blocks web thread. conn.request("POST", "/workflow.xml", request, headers) response = conn.getresponse() response_data = response.read() conn.close() # Do simple parse of response to see if export successful and provide user feedback. parser = SingleTagContentsParser('id') parser.feed(response_data) myexp_workflow_id = parser.tag_content workflow_list_str = " <br>Return to <a href='%s'>workflow list." % url_for(controller='workflows', action='list') if myexp_workflow_id: return trans.show_message( """Workflow '%s' successfully exported to myExperiment. <br/> <a href="http://%s/workflows/%s">Click here to view the workflow on myExperiment</a> %s """ % (stored.name, myexp_url, myexp_workflow_id, workflow_list_str), use_panels=True) else: return trans.show_error_message( "Workflow '%s' could not be exported to myExperiment. Error: %s %s" % (stored.name, response_data, workflow_list_str), use_panels=True)
def test_http_over_https(self): if self.scheme != 'https': return self.skip('skipped (not running HTTPS)... ') # Try connecting without SSL. conn = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) conn.putrequest('GET', '/', skip_host=True) conn.putheader('Host', self.HOST) conn.endheaders() response = conn.response_class(conn.sock, method='GET') try: response.begin() self.assertEqual(response.status, 400) self.body = response.read() self.assertBody('The client sent a plain HTTP request, but this ' 'server only speaks HTTPS on this port.') except socket.error: e = sys.exc_info()[1] # "Connection reset by peer" is also acceptable. if e.errno != errno.ECONNRESET: raise
def _checkURL(self, url): """ Check if the ``url`` is *alive* (i.e., remote server returns code 200(OK) or 401 (unauthorized)). """ try: p = urlparse(url) h = HTTPConnection(p[1]) h.putrequest('HEAD', p[2]) h.endheaders() r = h.getresponse() # CloudMan UI is pwd protected so include 401 if r.status in (200, 401): return True except Exception: # No response or no good response pass return False
def _findMatchUrl(self, tag): h3 = tag.find('h3') if not h3: return '' a = h3.find('a') url = a.attrs.get('href', '') # decode url host = parse.urlsplit(url).netloc path = url[len(parse.urljoin(url, '/')) - 1:] conn = HTTPConnection(host, timeout=10) conn.request('GET', path) req = conn.getresponse() r_url = req.getheader('Location') conn.close() return r_url
def _request(self, method, path, body=None): payload = None if body is not None: try: payload = json.dumps(body) except ValueError: raise ValueError("Failed to encode request body as JSON: {}".format( json.dumps(body, indent=2))) if isinstance(payload, text_type): payload = body.encode("utf-8") conn = HTTPConnection(self.host, self.port) try: conn.request(method, path, payload) yield conn.getresponse() finally: conn.close()
def _get_version(self, master): if master is not None: conn = None host, port = master.split(':', 2) port = int(port) try: conn = HTTPConnection(host, port, timeout=self._timeout) conn.request('GET', '/version') resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: return return json.loads(resp.read().decode('utf-8'))['version'] except Exception: logger.exception('Error') pass finally: if conn: conn.close()
def retry_http_ntlm_sspi_auth(self, host, req, authenticatehdr, auth_header): url = req.full_url scheme, _, host, path = url.split('/', 3) h = HTTPConnection(host) if scheme == 'http:' else HTTPSConnection( host) headers = dict(req.unredirected_hdrs) headers.update( dict((k, v) for k, v in req.headers.items() if k not in headers)) headers["Connection"] = "Keep-Alive" headers[auth_header] = "NTLM " + self.get_auth_req() h.request(req.get_method(), req.selector, req.data, headers) response = h.getresponse() response.fp = None # keep-alive ntlmauth = response.headers.get(authenticatehdr) if ntlmauth is not None and ntlmauth.startswith('NTLM '): challenge = ntlmauth[5:] challenge_response = self.create_challenge_response(challenge) if challenge_response is None: _log.warning('Failed to authenticate using NTLM') return None headers["Connection"] = "Close" headers[auth_header] = "NTLM " + challenge_response h.request(req.get_method(), req.selector, req.data, headers) response = h.getresponse() return addinfourl(response, response.msg, req.get_full_url(), response.status)
def _checkURL(self, url): """ Check if the ``url`` is *alive* (i.e., remote server returns code 200(OK) or 401 (unauthorized)). """ try: p = urlparse(url) h = HTTPConnection(p[1]) h.putrequest('HEAD', p[2]) h.endheaders() r = h.getresponse() if r.status in (200, 401): # CloudMan UI is pwd protected so include 401 return True except Exception: # No response or no good response pass return False
def send_email(request): try: recipients = request.GET['to'].split(',') url = request.GET['url'] proto, server, path, query, frag = urlsplit(url) if query: path += '?' + query conn = HTTPConnection(server) conn.request('GET', path) try: # Python 2.7+, use buffering of HTTP responses resp = conn.getresponse(buffering=True) except TypeError: # Python 2.6 and older resp = conn.getresponse() assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason) rawData = resp.read() conn.close() message = MIMEMultipart() message['Subject'] = "Graphite Image" message['To'] = ', '.join(recipients) message['From'] = 'composer@%s' % gethostname() text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(), url)) image = MIMEImage(rawData) image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png")) message.attach(text) message.attach(image) s = SMTP(settings.SMTP_SERVER) s.sendmail('composer@%s' % gethostname(), recipients, message.as_string()) s.quit() return HttpResponse("OK") except: return HttpResponse(format_exc())
def test_no_content_length(self): # "The presence of a message-body in a request is signaled by the # inclusion of a Content-Length or Transfer-Encoding header field in # the request's message-headers." # # Send a message with neither header and no body. Even though # the request is of method POST, this should be OK because we set # request.process_request_body to False for our handler. c = self.make_connection() c.request('POST', '/no_body') response = c.getresponse() self.body = response.fp.read() self.status = str(response.status) self.assertStatus(200) self.assertBody(b'Hello world!') # Now send a message that has no Content-Length, but does send a body. # Verify that CP times out the socket and responds # with 411 Length Required. if self.scheme == 'https': c = HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) else: c = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) # `_get_content_length` is needed for Python 3.6+ with mock.patch.object( c, '_get_content_length', lambda body, method: None, create=True): # `_set_content_length` is needed for Python 2.7-3.5 with mock.patch.object(c, '_set_content_length', create=True): c.request('POST', '/') response = c.getresponse() self.body = response.fp.read() self.status = str(response.status) self.assertStatus(411)
def kill_server(ipport, ipport2server): server, number = get_server_number(ipport, ipport2server) err = Manager([server]).kill(number=number) if err: raise Exception('unable to kill %s' % (server if not number else '%s%s' % (server, number))) try_until = time() + 30 while True: try: conn = HTTPConnection(*ipport) conn.request('GET', '/') conn.getresponse() except Exception as err: break if time() > try_until: raise Exception( 'Still answering on %s:%s after 30 seconds' % ipport) sleep(0.1)
def _download_http_file(self, url, target_path): '''下载http文件 :param url: HTTP路径 :type url: string :param target_path: 下载到目标路径 :type target_path: string ''' url0 = url if url[:7] == 'http://': url = url[7:] pos = url.find('/') host = url[:pos] page = url[pos:] conn = HTTPConnection(host, port=80, timeout=60) # 60秒超时 conn.request('GET', page) res = conn.getresponse() if res.status != 200: raise RuntimeError('访问:%s 错误[HTTP错误码:%s]' % (url0, res.status)) data = res.read() conn.close() f = open(target_path, 'wb') f.write(data) f.close()
def request(self, sitename, path, method='GET', data=None): url = self.base_url + '/' + sitename + path path = '/' + sitename + path if isinstance(data, dict): for k in data.keys(): if data[k] is None: del data[k] if web.config.debug: web.ctx.infobase_req_count = 1 + web.ctx.get( 'infobase_req_count', 0) a = time.time() _path = path _data = data headers = {} if data: if isinstance(data, dict): data = dict( (web.safestr(k), web.safestr(v)) for k, v in data.items()) data = urlencode(data) headers['Content-Type'] = 'application/x-www-form-urlencoded' if method == 'GET': path += '?' + data data = None stats.begin("infobase", path=path, method=method, data=data) conn = HTTPConnection(self.base_url) env = web.ctx.get('env') or {} if self.auth_token: c = SimpleCookie() c['infobase_auth_token'] = quote(self.auth_token) cookie = c.output(header='').strip() headers['Cookie'] = cookie # pass the remote ip to the infobase server headers['X-REMOTE-IP'] = web.ctx.get('ip') try: conn.request(method, path, data, headers=headers) response = conn.getresponse() stats.end() except socket.error: stats.end(error=True) logger.error("Unable to connect to infobase server", exc_info=True) raise ClientException("503 Service Unavailable", "Unable to connect to infobase server") cookie = response.getheader('Set-Cookie') if cookie: c = SimpleCookie() c.load(cookie) if 'infobase_auth_token' in c: auth_token = c['infobase_auth_token'].value # The auth token will be in urlquoted form, unquote it before use. # Otherwise, it will be quoted twice this value is set as cookie. auth_token = auth_token and unquote(auth_token) self.set_auth_token(auth_token) if web.config.debug: b = time.time() print("%.02f (%s):" % (round(b - a, 2), web.ctx.infobase_req_count), response.status, method, _path, _data, file=web.debug) if response.status == 200: return response.read() else: self.handle_error("%d %s" % (response.status, response.reason), response.read())
def stop_server(self): """send QUIT request to http server running on localhost:<port>""" conn = HTTPConnection("127.0.0.1:{}".format(self.port)) conn.request("QUIT", "/") conn.getresponse()
def _makeConnection(self, host=None, port=None): h = HTTPConnection(host or self.LOCALHOST, port or self.port) self.addCleanup(h.close) return h
class MesosExecutorDriver(Process, ExecutorDriver): def __init__(self, executor): env = os.environ agent_endpoint = env['MESOS_AGENT_ENDPOINT'] framework_id = env['MESOS_FRAMEWORK_ID'] assert framework_id self.framework_id = dict(value=framework_id) executor_id = env['MESOS_EXECUTOR_ID'] self.executor_id = dict(value=executor_id) grace_shutdown_period = env.get('MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD') if grace_shutdown_period: self.grace_shutdown_period = parse_duration(grace_shutdown_period) else: self.grace_shutdown_period = 0.0 self.checkpoint = bool(env.get('MESOS_CHECKPOINT')) self.local = bool(env.get('MESOS_LOCAL')) self.executor = executor self.framework_info = None self.executor_info = None self.tasks = {} self.updates = {} self._conn = None super(MesosExecutorDriver, self).__init__(master=agent_endpoint) def _delay_kill(self): def _(): try: time.sleep(self.grace_shutdown_period) os.killpg(0, signal.SIGKILL) except Exception: logger.exception('Failed to force kill executor') t = Thread(target=_) t.daemon = True t.start() def gen_request(self): body = json.dumps( dict( type='SUBSCRIBE', framework_id=self.framework_id, executor_id=self.executor_id, subscribe=dict( unacknowledged_tasks=list(self.tasks.values()), unacknowledged_updates=list(self.updates.values()), ), )) request = ('POST /api/v1/executor HTTP/1.1\r\nHost: %s\r\n' 'Content-Type: application/json\r\n' 'Accept: application/json\r\n' 'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % ( self.master, len(body), body) return request.encode('utf-8') def on_close(self): if self._conn is not None: self._conn.close() self._conn = None self.version = None self.executor.disconnected(self) if not self.checkpoint: if not self.local: self._delay_kill() self.executor.shutdown(self) self.abort() def on_event(self, event): if 'type' in event: _type = event['type'].lower() if _type == 'shutdown': self.on_shutdown() return if _type == 'heartbeat': return if _type not in event: logger.error('Missing `%s` in event %s' % (_type, event)) return event = event[_type] func_name = 'on_%s' % (_type, ) func = getattr(self, func_name, None) if func is not None: func(event) else: logger.error('Unknown type:%s, event:%s' % (_type, event)) else: logger.error('Unknown event:%s' % (event, )) def on_subscribed(self, info): executor_info = info['executor_info'] framework_info = info['framework_info'] agent_info = info['agent_info'] assert executor_info['executor_id'] == self.executor_id assert framework_info['id'] == self.framework_id if self.executor_info is None or self.framework_info is None: self.executor_info = executor_info self.framework_info = framework_info self.executor.registered(self, executor_info, framework_info, agent_info) else: self.executor.reregistered(self, agent_info) def on_launch(self, event): task_info = event['task'] task_id = task_info['task_id']['value'] assert task_id not in self.tasks self.tasks[task_id] = task_info self.executor.launchTask(self, task_info) def on_kill(self, event): task_id = event['task_id'] self.executor.killTask(self, task_id) def on_acknowledged(self, event): task_id = event['task_id']['value'] uuid_ = uuid.UUID(bytes=a2b_base64(event['uuid'])) self.updates.pop(uuid_, None) self.tasks.pop(task_id, None) def on_message(self, event): data = event['data'] self.executor.frameworkMessage(self, data) def on_error(self, event): message = event['message'] self.executor.error(self, message) def on_shutdown(self): if not self.local: self._delay_kill() self.executor.shutdown(self) self.abort() def _get_conn(self): if not self.connected: return None if self._conn is not None: return self._conn host, port = self.master.split(':', 2) port = int(port) self._conn = HTTPConnection(host, port, timeout=1) return self._conn def _send(self, body, path='/api/v1/executor', method='POST', headers={}): conn = self._get_conn() if conn is None: raise RuntimeError('Not connected yet') if body != '': data = json.dumps(body).encode('utf-8') headers['Content-Type'] = 'application/json' else: data = '' stream_id = self.stream_id if stream_id: headers['Mesos-Stream-Id'] = stream_id try: conn.request(method, path, body=data, headers=headers) resp = conn.getresponse() except Exception: self._conn.close() self._conn = None raise if resp.status < 200 or resp.status >= 300: raise RuntimeError('Failed to send request code=%s, message=%s' % (resp.status, resp.read())) result = resp.read() if not result: return {} try: return json.loads(result.decode('utf-8')) except Exception: return {} def sendStatusUpdate(self, status): if 'timestamp' not in status: status['timestamp'] = int(time.time()) if 'uuid' not in status: status['uuid'] = b2a_base64(uuid.uuid4().bytes) if 'source' not in status: status['source'] = 'SOURCE_EXECUTOR' body = dict( type='UPDATE', executor_id=self.executor_id, framework_id=self.framework_id, update=dict(status=status, ), ) self._send(body) def sendFrameworkMessage(self, data): body = dict( type='MESSAGE', executor_id=self.executor_id, framework_id=self.framework_id, message=dict(data=data, ), ) self._send(body)
def putrequest(self, *args, **kw): self.url = args[1] response = self.fakedata.pop(0) # get first response self.sock = FakeSocket(response) # and set up a fake socket output.new() # as well as an output buffer HTTPConnection.putrequest(self, *args, **kw)
class MesosSchedulerDriver(Process, SchedulerDriver): _timeout = 10 def __init__(self, sched, framework, master_uri, use_addict=False, implicit_acknowledgements=True, principal=None, secret=None, failover=False, timeout=DAY): super(MesosSchedulerDriver, self).__init__(timeout=timeout) self.sched = sched self.master_uri = master_uri self._framework = framework self.detector = None self._conn = None self.version = None self._failover = failover self._dict_cls = Dict if use_addict else dict self.implicit_acknowledgements = implicit_acknowledgements if principal is not None and secret is not None: self._basic_credential = 'Basic %s' % ( b2a_base64( ('%s:%s' % (principal, secret)).encode('ascii') ).decode('ascii').strip() ) else: self._basic_credential = None @property def framework(self): framework = dict(self._framework) version = self.version and tuple( int(n) for n in self.version.split('.') ) capabilities = [ c for c in framework.get('capabilities', []) if c['type'] != 'GPU_RESOURCES' ] if version and version >= (1, 0, 0): capabilities.append(dict(type='GPU_RESOURCES')) if capabilities: framework['capabilities'] = capabilities else: framework.pop('capabilities', None) if 'failover_timeout' not in framework: framework['failover_timeout'] = 100 return framework @property def framework_id(self): id = self._framework.get('id') return id and id.get('value') @framework_id.setter def framework_id(self, id): self._framework['id'] = dict(value=id) def _get_version(self, master): if master is not None: conn = None host, port = master.split(':', 2) port = int(port) try: conn = HTTPConnection(host, port, timeout=self._timeout) conn.request('GET', '/version') resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: return return json.loads(resp.read().decode('utf-8'))['version'] except Exception: logger.exception('Error') pass finally: if conn: conn.close() def change_master(self, master): self.version = self._get_version(master) super(MesosSchedulerDriver, self).change_master(master) self._close() def start(self): super(MesosSchedulerDriver, self).start() uri = self.master_uri if uri.startswith('zk://') or uri.startswith('zoo://'): from .detector import MasterDetector self.detector = MasterDetector(uri[uri.index('://') + 3:], self) self.detector.start() else: if ':' not in uri: uri += ':5050' self.change_master(uri) def stop(self, failover=False): with self._lock: self._failover = failover detector = self.detector self.detector = None if detector: detector.stop() super(MesosSchedulerDriver, self).stop() def _shutdown(self): if not self._failover: try: self._teardown() except Exception: logger.exception('Failed to Teardown') def _get_conn(self): if not self.connected: return None if self._conn is not None: return self._conn host, port = self.master.split(':', 2) port = int(port) self._conn = HTTPConnection(host, port, timeout=self._timeout) return self._conn def _send(self, body, path='/api/v1/scheduler', method='POST', headers={}): with self._lock: conn = self._get_conn() if conn is None: raise RuntimeError('Not connected yet') if body != '': data = json.dumps(body).encode('utf-8') headers['Content-Type'] = 'application/json' else: data = b'' stream_id = self.stream_id if stream_id: headers['Mesos-Stream-Id'] = stream_id if self._basic_credential: headers['Authorization'] = self._basic_credential try: conn.request(method, path, body=data, headers=headers) resp = conn.getresponse() except Exception: self._close() raise if resp.status < 200 or resp.status >= 300: raise RuntimeError('Failed to send request %s: %s\n%s' % ( resp.status, resp.read(), data)) result = resp.read() if not result: return {} try: return json.loads(result.decode('utf-8')) except Exception: return {} def _teardown(self): if self.connected: framework_id = self.framework_id if framework_id: self._send(dict( type='TEARDOWN', framework_id=dict( value=framework_id, ), )) self._framework.pop('id', None) def acceptOffers(self, offer_ids, operations, filters=None): if not operations: return self.declineOffer(offer_ids, filters=filters) if not self.connected: return framework_id = self.framework_id assert framework_id accept = dict( offer_ids=[offer_ids] if isinstance(offer_ids, dict) else offer_ids, operations=operations, ) if filters is not None: accept['filters'] = filters body = dict( type='ACCEPT', framework_id=dict( value=framework_id, ), accept=accept, ) self._send(body) def acceptInverseOffers(self, offer_ids, filters=None): framework_id = self.framework_id assert framework_id accept_inverse_offers = dict( inverse_offer_ids=[offer_ids] if isinstance(offer_ids, dict) else offer_ids ) if filters is not None: accept_inverse_offers['filters'] = filters body = dict( type='ACCEPT_INVERSE_OFFERS', framework_id=dict( value=framework_id, ), accept_inverse_offers=accept_inverse_offers, ) self._send(body) def launchTasks(self, offer_ids, tasks, filters=None): if not tasks: return self.declineOffer(offer_ids, filters=filters) if not self.connected: return framework_id = self.framework_id assert framework_id operations = [dict( type='LAUNCH', launch=dict( task_infos=tasks ), )] self.acceptOffers(offer_ids, operations, filters=filters) def declineOffer(self, offer_ids, filters=None): if not self.connected: return framework_id = self.framework_id assert framework_id decline = dict( offer_ids=[offer_ids] if isinstance(offer_ids, dict) else offer_ids ) if filters is not None: decline['filters'] = filters body = dict( type='DECLINE', framework_id=dict( value=framework_id, ), decline=decline, ) self._send(body) def declineInverseOffer(self, offer_ids, filters=None): if not self.connected: return framework_id = self.framework_id assert framework_id decline_inverse_offers = dict( inverse_offer_ids=[offer_ids] if isinstance(offer_ids, dict) else offer_ids ) if filters is not None: decline_inverse_offers['filters'] = filters body = dict( type='DECLINE_INVERSE_OFFERS', framework_id=dict( value=framework_id, ), decline_inverse_offers=decline_inverse_offers, ) self._send(body) def reviveOffers(self, roles=()): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='REVIVE', framework_id=dict( value=framework_id, ), ) if roles: body['revive'] = dict(roles=list(roles)) self._send(body) def suppressOffers(self, roles=()): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='SUPPRESS', framework_id=dict( value=framework_id, ), ) if roles: body['suppress'] = dict(roles=list(roles)) self._send(body) def killTask(self, task_id): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='KILL', framework_id=dict( value=framework_id, ), kill=dict( task_id=task_id, ), ) self._send(body) def acknowledgeStatusUpdate(self, status): if self.connected and 'uuid' in status: framework_id = self.framework_id assert framework_id acknowledge = dict() acknowledge['agent_id'] = status['agent_id'] acknowledge['task_id'] = status['task_id'] acknowledge['uuid'] = status['uuid'] body = dict( type='ACKNOWLEDGE', framework_id=dict( value=framework_id, ), acknowledge=acknowledge, ) self._send(body) def acknowledgeOperationStatusUpdate(self, status): if self.connected and 'uuid' in status and 'operation_id' in status: framework_id = self.framework_id assert framework_id body = dict( type='ACKNOWLEDGE_OPERATION_STATUS', framework_id=dict( value=framework_id, ), acknowledge_operation_status=status, ) self._send(body) def reconcileTasks(self, tasks): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='RECONCILE', framework_id=dict( value=framework_id, ), reconcile=dict( tasks=[dict(task_id=task['task_id']) for task in tasks], ), ) self._send(body) def reconcileOperations(self, operations_): if not self.connected: return framework_id = self.framework_id assert framework_id operations = [] for op_ in operations_: op = dict( operation_id=op_['operation_id'] ) if 'agent_id' in op_: op['agent_id'] = op_['agent_id'] if 'resource_provider_id' in op_: op['resource_provider_id'] = op_['resource_provider_id'] operations.append(op) body = dict( type='RECONCILE_OPERATIONS', framework_id=dict( value=framework_id, ), reconcile_operations=dict( operations=operations, ), ) self._send(body) def sendFrameworkMessage(self, executor_id, agent_id, data): if not self.connected: return framework_id = self.framework_id assert framework_id message = dict( agent_id=agent_id, executor_id=executor_id, data=data, ) body = dict( type='MESSAGE', framework_id=dict( value=framework_id, ), message=message, ) self._send(body) def requestResources(self, requests): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='REQUEST', framework_id=dict( value=framework_id, ), request=dict( requests=requests, ), ) self._send(body) def onNewMasterDetectedMessage(self, data): master = None try: if isinstance(data, six.binary_type): data = data.decode('utf-8') parsed = json.loads(data) if parsed and "address" in parsed: ip = parsed["address"].get("ip") port = parsed["address"].get("port") if ip and port: master = "%s:%s" % (ip, port) except Exception: logger.exception("No JSON content, probably connecting " "to older Mesos version.") if master: self.change_master(master) def onNoMasterDetectedMessage(self): self.change_master(None) def gen_request(self): request = dict( type='SUBSCRIBE', subscribe=dict( framework_info=self.framework ), ) if 'id' in self._framework: request['framework_id'] = self._framework['id'] data = json.dumps(request) _authorization = '' if self._basic_credential is not None: _authorization = 'Authorization: %s\r\n' % ( self._basic_credential, ) request = ('POST /api/v1/scheduler HTTP/1.1\r\nHost: %s\r\n' 'Content-Type: application/json\r\n' 'Accept: application/json\r\n%s' 'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % ( self.master, _authorization, len(data), data ) return request.encode('utf-8') def _close(self): if self._conn is not None: self._conn.close() self._conn = None def on_close(self): self._close() self.sched.disconnected(self) def on_subscribed(self, info): reregistered = (self.framework_id is not None) self.framework_id = info['framework_id']['value'] hostname, port = self.master.split(':', 2) port = int(port) master_info = dict( hostname=hostname, port=port, ) if self.version: master_info['version'] = self.version elif 'master_info' in info and 'version' in info['master_info']: master_info['version'] = info['master_info']['version'] if reregistered: self.sched.reregistered(self, self._dict_cls(master_info)) else: framework_id = dict( value=self.framework_id ) self.sched.registered( self, self._dict_cls(framework_id), self._dict_cls(master_info) ) def on_offers(self, event): offers = event.get('offers', []) if offers: self.sched.resourceOffers( self, [self._dict_cls(offer) for offer in offers] ) version = self.version and tuple( int(n) for n in self.version.split('.') ) if not (version and version >= (1, 0, 0)): self.on_inverse_offers(event) def on_inverse_offers(self, event): inverse_offers = event.get('inverse_offers', []) if inverse_offers: self.sched.inverseOffers( self, [self._dict_cls(offer) for offer in inverse_offers] ) def on_rescind(self, event): offer_id = event['offer_id'] self.sched.offerRescinded(self, self._dict_cls(offer_id)) def on_rescind_inverse_offer(self, event): inverse_offer_id = event['inverse_offer_id'] self.sched.inverseOfferRescinded( self, self._dict_cls(inverse_offer_id) ) def on_update(self, event): status = event['status'] self.sched.statusUpdate(self, self._dict_cls(status)) if self.implicit_acknowledgements: self.acknowledgeStatusUpdate(status) def on_update_operation_status(self, event): status = event['status'] self.sched.operationStatusUpdate(self, self._dict_cls(status)) if self.implicit_acknowledgements: self.acknowledgeOperationStatusUpdate(status) def on_message(self, message): executor_id = message['executor_id'] agent_id = message['agent_id'] data = message['data'] self.sched.frameworkMessage( self, self._dict_cls(executor_id), self._dict_cls(agent_id), data ) def on_failure(self, failure): agent_id = failure['agent_id'] if 'executor_id' not in failure: self.sched.slaveLost(self, self._dict_cls(agent_id)) else: self.sched.executorLost( self, self._dict_cls(failure['executor_id']), self._dict_cls(agent_id), failure['status'] ) def on_error(self, event): message = event['message'] self.sched.error(self, message) def on_heartbeat(self): self.sched.processHeartBeat() def on_event(self, event): if 'type' in event: _type = event['type'].lower() if _type == 'heartbeat': self.on_heartbeat() return if _type not in event: logger.error( 'Missing `%s` in event %s' % (_type, event)) return event = event[_type] func_name = 'on_%s' % (_type,) func = getattr(self, func_name, None) if func is not None: func(event) else: logger.error('Unknown type:%s, event:%s' % (_type, event)) else: logger.error('Unknown event:%s' % (event,))
class MesosExecutorDriver(Process, ExecutorDriver): _timeout = 10 def __init__(self, executor, use_addict=False): env = os.environ agent_endpoint = env['MESOS_AGENT_ENDPOINT'] super(MesosExecutorDriver, self).__init__(master=agent_endpoint) framework_id = env['MESOS_FRAMEWORK_ID'] assert framework_id self.framework_id = dict(value=framework_id) executor_id = env['MESOS_EXECUTOR_ID'] self.executor_id = dict(value=executor_id) grace_shutdown_period = env.get('MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD') if grace_shutdown_period: self.grace_shutdown_period = parse_duration(grace_shutdown_period) else: self.grace_shutdown_period = 0.0 self.checkpoint = bool(env.get('MESOS_CHECKPOINT')) self.local = bool(env.get('MESOS_LOCAL')) self.executor = executor self.framework_info = None self.executor_info = None self.tasks = {} self.updates = {} self._conn = None self._dict_cls = Dict if use_addict else dict def _delay_kill(self): def _(): try: time.sleep(self.grace_shutdown_period) os.killpg(0, signal.SIGKILL) except Exception: logger.exception('Failed to force kill executor') t = Thread(target=_) t.daemon = True t.start() def gen_request(self): body = json.dumps(dict( type='SUBSCRIBE', framework_id=self.framework_id, executor_id=self.executor_id, subscribe=dict( unacknowledged_tasks=list(self.tasks.values()), unacknowledged_updates=list(self.updates.values()), ), )) request = ('POST /api/v1/executor HTTP/1.1\r\nHost: %s\r\n' 'Content-Type: application/json\r\n' 'Accept: application/json\r\n' 'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % ( self.master, len(body), body ) return request.encode('utf-8') def on_close(self): if self._conn is not None: self._conn.close() self._conn = None self.version = None self.executor.disconnected(self) if not self.checkpoint: if not self.local: self._delay_kill() self.executor.shutdown(self) self.abort() def on_event(self, event): if 'type' in event: _type = event['type'].lower() if _type == 'shutdown': self.on_shutdown() return if _type == 'heartbeat': return if _type not in event: logger.error( 'Missing `%s` in event %s' % (_type, event)) return event = event[_type] func_name = 'on_%s' % (_type,) func = getattr(self, func_name, None) if func is not None: func(event) else: logger.error('Unknown type:%s, event:%s' % (_type, event)) else: logger.error('Unknown event:%s' % (event,)) def on_subscribed(self, info): executor_info = info['executor_info'] framework_info = info['framework_info'] agent_info = info['agent_info'] assert executor_info['executor_id'] == self.executor_id assert framework_info['id'] == self.framework_id if self.executor_info is None or self.framework_info is None: self.executor_info = executor_info self.framework_info = framework_info self.executor.registered( self, self._dict_cls(executor_info), self._dict_cls(framework_info), self._dict_cls(agent_info) ) else: self.executor.reregistered(self, self._dict_cls(agent_info)) def on_launch(self, event): task_info = event['task'] task_id = task_info['task_id']['value'] assert task_id not in self.tasks self.tasks[task_id] = task_info self.executor.launchTask(self, self._dict_cls(task_info)) def on_kill(self, event): task_id = event['task_id'] self.executor.killTask(self, self._dict_cls(task_id)) def on_acknowledged(self, event): task_id = event['task_id']['value'] uuid_ = uuid.UUID(bytes=decode_data(event['uuid'])) self.updates.pop(uuid_, None) self.tasks.pop(task_id, None) def on_message(self, event): data = event['data'] self.executor.frameworkMessage(self, data) def on_error(self, event): message = event['message'] self.executor.error(self, message) def on_shutdown(self): if not self.local: self._delay_kill() self.executor.shutdown(self) self.abort() def _get_conn(self): if not self.connected: return None if self._conn is not None: return self._conn host, port = self.master.split(':', 2) port = int(port) self._conn = HTTPConnection(host, port, timeout=self._timeout) return self._conn def _send(self, body, path='/api/v1/executor', method='POST', headers={}): with self._lock: conn = self._get_conn() if conn is None: raise RuntimeError('Not connected yet') if body != '': data = json.dumps(body).encode('utf-8') headers['Content-Type'] = 'application/json' else: data = '' stream_id = self.stream_id if stream_id: headers['Mesos-Stream-Id'] = stream_id try: conn.request(method, path, body=data, headers=headers) resp = conn.getresponse() except Exception: self._conn.close() self._conn = None raise if resp.status < 200 or resp.status >= 300: raise RuntimeError( 'Failed to send request code=%s, message=%s' % ( resp.status, resp.read() ) ) result = resp.read() if not result: return {} try: return json.loads(result.decode('utf-8')) except Exception: return {} def sendStatusUpdate(self, status): if 'timestamp' not in status: status['timestamp'] = int(time.time()) if 'uuid' not in status: status['uuid'] = encode_data(uuid.uuid4().bytes) if 'source' not in status: status['source'] = 'SOURCE_EXECUTOR' body = dict( type='UPDATE', executor_id=self.executor_id, framework_id=self.framework_id, update=dict( status=status, ), ) self._send(body) def sendFrameworkMessage(self, data): body = dict( type='MESSAGE', executor_id=self.executor_id, framework_id=self.framework_id, message=dict( data=data, ), ) self._send(body)
class HTTPWireProtocol(object): """ Transports messages (commands and responses) over the WebDriver wire protocol. Complex objects, such as ``webdriver.Element``, ``webdriver.Frame``, and ``webdriver.Window`` are by default not marshaled to enable use of `session.transport.send` in WPT tests:: session = webdriver.Session("127.0.0.1", 4444) response = transport.send("GET", "element/active", None) print response.body["value"] # => {u'element-6066-11e4-a52e-4f735466cecf': u'<uuid>'} Automatic marshaling is provided by ``webdriver.protocol.Encoder`` and ``webdriver.protocol.Decoder``, which can be passed in to ``HTTPWireProtocol.send`` along with a reference to the current ``webdriver.Session``:: session = webdriver.Session("127.0.0.1", 4444) response = transport.send("GET", "element/active", None, encoder=protocol.Encoder, decoder=protocol.Decoder, session=session) print response.body["value"] # => webdriver.Element """ def __init__(self, host, port, url_prefix="/"): """ Construct interface for communicating with the remote server. :param url: URL of remote WebDriver server. :param wait: Duration to wait for remote to appear. """ self.host = host self.port = port self.url_prefix = url_prefix self._conn = None self._last_request_is_blocked = False def __del__(self): self.close() def close(self): """Closes the current HTTP connection, if there is one.""" if self._conn: self._conn.close() @property def connection(self): """Gets the current HTTP connection, or lazily creates one.""" if not self._conn: conn_kwargs = {} if not PY3: conn_kwargs["strict"] = True # We are not setting an HTTP timeout other than the default when the # connection its created. The send method has a timeout value if needed. self._conn = HTTPConnection(self.host, self.port, **conn_kwargs) return self._conn def url(self, suffix): """ From the relative path to a command end-point, craft a full URL suitable to be used in a request to the HTTPD. """ return urlparse.urljoin(self.url_prefix, suffix) def send(self, method, uri, body=None, headers=None, encoder=json.JSONEncoder, decoder=json.JSONDecoder, timeout=None, **codec_kwargs): """ Send a command to the remote. The request `body` must be JSON serialisable unless a custom `encoder` has been provided. This means complex objects such as ``webdriver.Element``, ``webdriver.Frame``, and `webdriver.Window`` are not automatically made into JSON. This behaviour is, however, provided by ``webdriver.protocol.Encoder``, should you want it. Similarly, the response body is returned au natural as plain JSON unless a `decoder` that converts web element references to ``webdriver.Element`` is provided. Use ``webdriver.protocol.Decoder`` to achieve this behaviour. The client will attempt to use persistent HTTP connections. :param method: `GET`, `POST`, or `DELETE`. :param uri: Relative endpoint of the requests URL path. :param body: Body of the request. Defaults to an empty dictionary if ``method`` is `POST`. :param headers: Additional dictionary of headers to include in the request. :param encoder: JSON encoder class, which defaults to ``json.JSONEncoder`` unless specified. :param decoder: JSON decoder class, which defaults to ``json.JSONDecoder`` unless specified. :param codec_kwargs: Surplus arguments passed on to `encoder` and `decoder` on construction. :return: Instance of ``webdriver.transport.Response`` describing the HTTP response received from the remote end. :raises ValueError: If `body` or the response body are not JSON serialisable. """ if body is None and method == "POST": body = {} payload = None if body is not None: try: payload = json.dumps(body, cls=encoder, **codec_kwargs) except ValueError: raise ValueError("Failed to encode request body as JSON:\n" "%s" % json.dumps(body, indent=2)) # When the timeout triggers, the TestRunnerManager thread will reuse # this connection to check if the WebDriver its alive and we may end # raising an httplib.CannotSendRequest exception if the WebDriver is # not responding and this httplib.request() call is blocked on the # runner thread. We use the boolean below to check for that and restart # the connection in that case. self._last_request_is_blocked = True response = self._request(method, uri, payload, headers, timeout=None) self._last_request_is_blocked = False return Response.from_http(response, decoder=decoder, **codec_kwargs) def _request(self, method, uri, payload, headers=None, timeout=None): if isinstance(payload, text_type): payload = payload.encode("utf-8") if headers is None: headers = {} headers.update({"Connection": "keep-alive"}) url = self.url(uri) if self._last_request_is_blocked or self._has_unread_data(): self.close() self.connection.request(method, url, payload, headers) # timeout for request has to be set just before calling httplib.getresponse() # and the previous value restored just after that, even on exception raised try: if timeout: previous_timeout = self._conn.gettimeout() self._conn.settimeout(timeout) response = self.connection.getresponse() finally: if timeout: self._conn.settimeout(previous_timeout) return response def _has_unread_data(self): return self._conn and self._conn.sock and select.select([self._conn.sock], [], [], 0)[0]
class WebcamStreamBase(VideoStream): request_headers = { 'Accept': '*/*', } def __init__(self, url, max_rate=3.0, rate_bucket_size=None, socket_timeout=10, user_agent=DEFAULT_USER_AGENT): self.url = url netloc, self.path = _parse_url(url) self.conn = HTTPConnection(netloc, timeout=socket_timeout) self.request_headers = self.request_headers.copy() self.request_headers['User-Agent'] = user_agent self.stream = None self.rate_limiter = BucketRateLimiter(max_rate, rate_bucket_size) self.open_rate_limiter = BackoffRateLimiter(socket_timeout) def __repr__(self): return u"<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self.url) @classmethod def from_settings(cls, settings, prefix='webcam.', **defaults): config = config_from_settings(settings, prefix=prefix, subprefix=cls.settings_subprefix, **defaults) return cls(**config) def close(self): if self.stream: self.stream = None if self.conn: self.conn.close() self.conn = None @property def closed(self): return not self.conn def next(self): # FIXME: check closed more often? if self.closed: raise StopIteration() next(self.rate_limiter) try: if self.stream is None: next(self.open_rate_limiter) self.stream = self._open_stream() frame = next(self.stream) self.open_rate_limiter.reset() return frame except Exception as ex: self.stream = None log.warn("Streaming failed: %s", text_type(ex) or repr(ex)) self.conn.close() return None
def export_to_myexp(self, trans, id, myexp_username, myexp_password): """ Exports a workflow to myExperiment website. """ trans.workflow_building_mode = workflow_building_modes.ENABLED stored = self.get_stored_workflow(trans, id, check_ownership=False, check_accessible=True) # Convert workflow to dict. workflow_dict = self._workflow_to_dict(trans, stored) # # Create and submit workflow myExperiment request. # # Create workflow content JSON. workflow_content = json.dumps(workflow_dict, indent=4, sort_keys=True) # Create myExperiment request. request_raw = trans.fill_template( "workflow/myexp_export.mako", workflow_name=workflow_dict['name'], workflow_description=workflow_dict['annotation'], workflow_content=workflow_content, workflow_svg=self._workflow_to_svg_canvas(trans, stored).tostring()) # strip() b/c myExperiment XML parser doesn't allow white space before XML; utf-8 handles unicode characters. request = unicodify(request_raw.strip(), 'utf-8') # Do request and get result. auth_header = base64.b64encode('%s:%s' % (myexp_username, myexp_password)) headers = { "Content-type": "text/xml", "Accept": "text/xml", "Authorization": "Basic %s" % auth_header } myexp_url = trans.app.config.get("myexperiment_url", self.__myexp_url) conn = HTTPConnection(myexp_url) # NOTE: blocks web thread. conn.request("POST", "/workflow.xml", request, headers) response = conn.getresponse() response_data = response.read() conn.close() # Do simple parse of response to see if export successful and provide user feedback. parser = SingleTagContentsParser('id') parser.feed(response_data) myexp_workflow_id = parser.tag_content workflow_list_str = " <br>Return to <a href='%s'>workflow list." % url_for( controller='workflows', action='list') if myexp_workflow_id: return trans.show_message( """Workflow '%s' successfully exported to myExperiment. <br/> <a href="http://%s/workflows/%s">Click here to view the workflow on myExperiment</a> %s """ % (stored.name, myexp_url, myexp_workflow_id, workflow_list_str), use_panels=True) else: return trans.show_error_message( "Workflow '%s' could not be exported to myExperiment. Error: %s %s" % (stored.name, response_data, workflow_list_str), use_panels=True)
class MesosOperatorDaemonDriver(OperatorDaemonDriver): _timeout = 10 def __init__(self, daemon_uri): self.init(daemon_uri) def init(self, daemon_uri): """ :param daemon_uri: masterHost:5050 or agentHost:5051 """ self._daemon = daemon_uri self._conn = None def _get_conn(self): if self._conn is not None: return self._conn host, port = self._daemon.split(':', 2) port = int(port) self._conn = HTTPConnection(host, port, timeout=self._timeout) return self._conn def _send(self, body, path='/api/v1/operator', method='POST', headers={}): with self._lock: conn = self._get_conn() if conn is None: raise RuntimeError('Not connected yet') if body != '': data = json.dumps(body).encode('utf-8') headers['Content-Type'] = 'application/json' else: data = '' # stream_id = self.stream_id # if stream_id: # headers['Mesos-Stream-Id'] = stream_id try: conn.request(method, path, body=data, headers=headers) resp = conn.getresponse() except Exception: self._conn.close() self._conn = None raise if resp.status < 200 or resp.status >= 300: raise RuntimeError( 'Failed to send request code=%s, message=%s' % ( resp.status, resp.read() ) ) result = resp.read() if not result: return {} try: return json.loads(result.decode('utf-8')) except Exception: return {} def getHealth(self): body = dict( type='GET_HEALTH', ) return self._send(body) def getFlags(self): body = dict( type='GET_FLAGS', ) return self._send(body) def getVersion(self): body = dict( type='GET_VERSION', ) return self._send(body) def getMetrics(self, timeout): body = dict( type='GET_METRICS', get_metrics=dict( timeout=dict( nanoseconds=timeout, ) ) ) return self._send(body) def getLoggingLevel(self): body = dict( type='GET_LOGGING_LEVEL', ) return self._send(body) def setLoggingLevel(self, level, duration): body = dict( type='SET_LOGGING_LEVEL', set_logging_level=dict( duration=dict( nanoseconds=duration, ), level=level, ) ) return self._send(body) def listFiles(self, path): body = dict( type='LIST_FILES', list_files=dict( path=path, ), ) return self._send(body) def readFile(self, path, length, offset): body = dict( type='READ_FILE', read_file=dict( length=length, offset=offset, path=path, ) ) return self._send(body) def getState(self): body = dict( type='GET_STATE', ) return self._send(body) def getFrameworks(self): body = dict( type='GET_FRAMEWORKS', ) return self._send(body) def getExecutors(self): body = dict( type='GET_EXECUTORS', ) return self._send(body) def getTasks(self): body = dict( type='GET_TASKS', ) return self._send(body)
class MesosSchedulerDriver(Process, SchedulerDriver): def __init__(self, sched, framework, master_uri): super(MesosSchedulerDriver, self).__init__() self.sched = sched self.master_uri = master_uri self._framework = framework self.detector = None self._conn = None self.version = None @property def framework(self): framework = dict(self._framework) version = self.version and tuple( int(n) for n in self.version.split('.')) capabilities = [ c for c in framework.get('capabilities', []) if c['type'] != 'GPU_RESOURCES' ] if version and version >= (1, 0, 0): capabilities.append(dict(type='GPU_RESOURCES')) if capabilities: framework['capabilities'] = capabilities else: framework.pop('capabilities', None) return framework @property def framework_id(self): return self._framework.get('id') @framework_id.setter def framework_id(self, id): self._framework['id'] = id def _get_version(self, master): if master is not None: conn = None host, port = master.split(':', 2) port = int(port) try: conn = HTTPConnection(host, port, timeout=1) conn.request('GET', '/version') resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: return return json.loads(resp.read())['version'] except Exception: logger.exception('Error') pass finally: if conn: conn.close() def change_master(self, master): self.version = self._get_version(master) super(MesosSchedulerDriver, self).change_master(master) if self._conn is not None: self._conn.close() self._conn = None def start(self): super(MesosSchedulerDriver, self).start() uri = self.master_uri if uri.startswith('zk://') or uri.startswith('zoo://'): from .detector import MasterDetector self.detector = MasterDetector(uri[uri.index('://') + 3:], self) self.detector.start() else: if ':' not in uri: uri += ':5050' self.change_master(uri) def stop(self, failover=False): if not failover: try: self._teardown() except Exception: logger.exception('Failed to Teardown') if self.detector: self.detector.stop() super(MesosSchedulerDriver, self).stop() def _get_conn(self): if not self.connected: return None if self._conn is not None: return self._conn host, port = self.master.split(':', 2) port = int(port) self._conn = HTTPConnection(host, port, timeout=1) return self._conn def _send(self, body, path='/api/v1/scheduler', method='POST', headers={}): conn = self._get_conn() if conn is None: raise RuntimeError('Not connected yet') if body != '': data = json.dumps(body).encode('utf-8') headers['Content-Type'] = 'application/json' else: data = '' stream_id = self.stream_id if stream_id: headers['Mesos-Stream-Id'] = stream_id try: conn.request(method, path, body=data, headers=headers) resp = conn.getresponse() except Exception: self._conn.close() self._conn = None raise if resp.status < 200 or resp.status >= 300: raise RuntimeError('Failed to send request %s: %s\n%s' % (resp.status, resp.read(), data)) result = resp.read() if not result: return {} try: return json.loads(result.decode('utf-8')) except Exception: return {} def _teardown(self): framework_id = self.framework_id if framework_id: self._send( dict( type='TEARDOWN', framework_id=dict(value=framework_id, ), )) def acceptOffers(self, offer_ids, operations, filters=None): if not operations: return self.declineOffer(offer_ids, filters=filters) framework_id = self.framework_id assert framework_id accept = dict( offer_ids=offer_ids, operations=operations, ) if filters is not None: accept['filters'] = filters body = dict( type='ACCEPT', framework_id=dict(value=framework_id, ), accept=accept, ) self._send(body) def launchTasks(self, offer_ids, tasks, filters=None): if not tasks: return self.declineOffer(offer_ids, filters=filters) framework_id = self.framework_id assert framework_id operations = [dict( type='LAUNCH', launch=dict(task_infos=tasks), )] self.acceptOffers(offer_ids, operations, filters=filters) def declineOffer(self, offer_ids, filters=None): framework_id = self.framework_id assert framework_id decline = dict(offer_ids=offer_ids, ) if filters is not None: decline['filters'] = filters body = dict( type='DECLINE', framework_id=dict(value=framework_id, ), decline=decline, ) self._send(body) def reviveOffers(self): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='REVIVE', framework_id=dict(value=framework_id, ), ) self._send(body) def killTask(self, task_id): framework_id = self.framework_id assert framework_id body = dict( type='KILL', framework_id=dict(value=framework_id, ), kill=dict(task_id=task_id, ), ) self._send(body) def acknowledgeStatusUpdate(self, status): if 'uuid' in status: framework_id = self.framework_id assert framework_id acknowledge = dict() acknowledge['agent_id'] = status['agent_id'] acknowledge['task_id'] = status['task_id'] acknowledge['uuid'] = status['uuid'] body = dict( type='ACKNOWLEDGE', framework_id=dict(value=framework_id, ), acknowledge=acknowledge, ) self._send(body) def reconcileTasks(self, tasks): framework_id = self.framework_id assert framework_id body = dict( type='RECONCILE', framework_id=dict(value=framework_id, ), reconcile=dict( tasks=[dict(task_id=task['task_id']) for task in tasks], ), ) self._send(body) def sendFrameworkMessage(self, executor_id, agent_id, data): framework_id = self.framework_id assert framework_id message = dict( agent_id=agent_id, executor_id=executor_id, data=b2a_base64(data.encode('utf-8')).rstrip(), ) body = dict( type='MESSAGE', framework_id=dict(value=framework_id, ), message=message, ) self._send(body) def requestResources(self, requests): framework_id = self.framework_id assert framework_id body = dict( type='REQUEST', framework_id=dict(value=framework_id, ), request=dict(requests=requests, ), ) self._send(body) def onNewMasterDetectedMessage(self, data): master = None try: parsed = json.loads(data) if parsed and "address" in parsed: ip = parsed["address"].get("ip") port = parsed["address"].get("port") if ip and port: master = "%s:%s" % (ip, port) except Exception: logger.exception("No JSON content, probably connecting " "to older Mesos version.") if master: self.change_master(master) def onNoMasterDetectedMessage(self): self.change_master(None) def gen_request(self): data = json.dumps( dict( type='SUBSCRIBE', subscribe=dict(framework_info=self.framework), )) request = ('POST /api/v1/scheduler HTTP/1.1\r\nHost: %s\r\n' 'Content-Type: application/json\r\n' 'Accept: application/json\r\n' 'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % ( self.master, len(data), data) return request.encode('utf-8') def on_close(self): if self._conn is not None: self._conn.close() self._conn = None self.sched.disconnected(self) def on_subscribed(self, info): reregistered = (self.framework_id is not None) self.framework_id = info['framework_id']['value'] hostname, port = self.master.split(':', 2) port = int(port) master_info = dict( hostname=hostname, port=port, ) if self.version: master_info['version'] = self.version if reregistered: self.sched.reregistered(self, master_info) else: framework_id = dict(value=self.framework_id) self.sched.registered(self, framework_id, master_info) def on_offers(self, event): offers = event['offers'] self.sched.resourceOffers(self, offers) def on_rescind(self, event): offer_id = event['offer_id'] self.sched.offerRescinded(self, offer_id) def on_update(self, event): status = event['status'] self.sched.statusUpdate(self, status) self.acknowledgeStatusUpdate(status) def on_message(self, message): executor_id = message['executor_id'] agent_id = message['agent_id'] data = message['data'] self.sched.frameworkMessage(self, executor_id, agent_id, data) def on_failure(self, failure): agent_id = failure['agent_id'] if 'executor_id' not in failure: self.sched.slaveLost(self, agent_id) else: self.sched.executorLost(self, failure['executor_id'], agent_id, failure['status']) def on_error(self, event): message = event['message'] self.sched.error(self, message) def on_event(self, event): if 'type' in event: _type = event['type'].lower() if _type == 'heartbeat': return if _type not in event: logger.error('Missing `%s` in event %s' % (_type, event)) return event = event[_type] func_name = 'on_%s' % (_type, ) func = getattr(self, func_name, None) if func is not None: func(event) else: logger.error('Unknown type:%s, event:%s' % (_type, event)) else: logger.error('Unknown event:%s' % (event, ))
def retry_using_http_NTLM_auth(self, req, auth_header_field, realm, headers): user, pw = self.passwd.find_user_password(realm, req.get_full_url()) if pw is not None: user_parts = user.split('\\', 1) if len(user_parts) == 1: UserName = user_parts[0] DomainName = '' type1_flags = ntlm.NTLM_TYPE1_FLAGS & ~ntlm.NTLM_NegotiateOemDomainSupplied else: DomainName = user_parts[0].upper() UserName = user_parts[1] type1_flags = ntlm.NTLM_TYPE1_FLAGS # ntlm secures a socket, so we must use the same socket for the complete handshake headers = dict(req.headers) headers.update(req.unredirected_hdrs) auth = 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(user, type1_flags).decode("utf-8") if req.headers.get(self.auth_header, None) == auth: return None headers[self.auth_header] = auth host = req.host if not host: raise urllib.request.URLError('no host given') h = None if req.get_full_url().startswith('https://'): h = HTTPSConnection(host) # will parse host:port else: h = HTTPConnection(host) # will parse host:port h.set_debuglevel(self._debuglevel) # we must keep the connection because NTLM authenticates the connection, not single requests headers["Connection"] = "Keep-Alive" headers = dict((name.title(), val) for name, val in headers.items()) # For some reason, six doesn't do this translation correctly # TODO rsanders low - find bug in six & fix it try: selector = req.selector except AttributeError: selector = req.get_selector() h.request(req.get_method(), selector, req.data, headers) r = h.getresponse() r.begin() r._safe_read(int(r.getheader('content-length'))) if r.getheader('set-cookie'): # this is important for some web applications that store authentication-related info in cookies (it took a long time to figure out) headers['Cookie'] = r.getheader('set-cookie') r.fp = None # remove the reference to the socket, so that it can not be closed by the response object (we want to keep the socket open) auth_header_value = r.getheader(auth_header_field, None) # some Exchange servers send two WWW-Authenticate headers, one with the NTLM challenge # and another with the 'Negotiate' keyword - make sure we operate on the right one m = re.match('(NTLM [A-Za-z0-9+\-/=]+)', auth_header_value) if m: auth_header_value, = m.groups() (ServerChallenge, NegotiateFlags) = ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value[5:]) auth = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, UserName, DomainName, pw, NegotiateFlags).decode("utf-8") headers[self.auth_header] = auth headers["Connection"] = "Close" headers = dict((name.title(), val) for name, val in headers.items()) try: h.request(req.get_method(), selector, req.data, headers) # none of the configured handlers are triggered, for example redirect-responses are not handled! response = h.getresponse() def notimplemented(): raise NotImplementedError response.readline = notimplemented infourl = urllib.response.addinfourl(response, response.msg, req.get_full_url()) infourl.code = response.status infourl.msg = response.reason return infourl except socket.error as err: raise urllib.error.URLError(err) else: return None
def __init__(self, uri, basepath=None): self.basepath = basepath self.mimetype = None self.file = None self.data = None self.uri = None self.local = None self.tmp_file = None uri = uri or str() uri = uri.encode('utf-8') log.debug("FileObject %r, Basepath: %r", uri, basepath) # Data URI if isinstance(uri, binary_type): uri = uri.decode('utf-8') if uri.startswith("data:"): m = _rx_datauri.match(uri) self.mimetype = m.group("mime") self.data = base64.decodestring(m.group("data")) else: # Check if we have an external scheme if basepath and not urlparse(uri).scheme: urlParts = urlparse(basepath) else: urlParts = urlparse(uri) log.debug("URLParts: %r", urlParts) if urlParts.scheme == 'file': if basepath and uri.startswith('/'): uri = urljoin(basepath, uri[1:]) urlResponse = urlopen(uri) self.mimetype = urlResponse.info().get("Content-Type", '').split(";")[0] self.uri = urlResponse.geturl() self.file = urlResponse # Drive letters have len==1 but we are looking # for things like http: elif urlParts.scheme in ('http', 'https'): # External data if basepath: uri = urljoin(basepath, uri) #path = urlsplit(url)[2] #mimetype = getMimeType(path) # Using HTTPLIB parts = urlsplit(uri[uri.find("//"):]) server, path = parts.netloc, parts.path if uri.startswith("https://"): conn = HTTPSConnection(server) else: conn = HTTPConnection(server) conn.request("GET", path) r1 = conn.getresponse() # log.debug("HTTP %r %r %r %r", server, path, uri, r1) if (r1.status, r1.reason) == (200, "OK"): self.mimetype = r1.getheader("Content-Type", '').split(";")[0] self.uri = uri if r1.getheader("content-encoding") == "gzip": import gzip self.file = gzip.GzipFile(mode="rb", fileobj=BytesIO(r1.read())) else: self.file = r1 else: try: urlResponse = urlopen(uri) except HTTPError: return self.mimetype = urlResponse.info().get("Content-Type", '').split(";")[0] self.uri = urlResponse.geturl() self.file = urlResponse else: # Local data if basepath: uri = os.path.normpath(os.path.join(basepath, uri)) if os.path.isfile(uri): self.uri = uri self.local = uri self.setMimeTypeByName(uri) self.file = open(uri, "rb")
def __init__(self, host, *fakedata): HTTPConnection.__init__(self, host) self.fakedata = list(fakedata)
class MesosSchedulerDriver(Process, SchedulerDriver): _timeout = 10 def __init__(self, sched, framework, master_uri, use_addict=False, implicit_acknowledgements=True, principal=None, secret=None, failover=False, timeout=DAY): super(MesosSchedulerDriver, self).__init__(timeout=timeout) self.sched = sched self.master_uri = master_uri self._framework = framework self.detector = None self._conn = None self.version = None self._failover = failover self._dict_cls = Dict if use_addict else dict self.implicit_acknowledgements = implicit_acknowledgements if principal is not None and secret is not None: self._basic_credential = 'Basic %s' % (b2a_base64( ('%s:%s' % (principal, secret)).encode('ascii')).decode('ascii').strip()) else: self._basic_credential = None @property def framework(self): framework = dict(self._framework) version = self.version and tuple( int(n) for n in self.version.split('.')) capabilities = [ c for c in framework.get('capabilities', []) if c['type'] != 'GPU_RESOURCES' ] if version and version >= (1, 0, 0): capabilities.append(dict(type='GPU_RESOURCES')) if capabilities: framework['capabilities'] = capabilities else: framework.pop('capabilities', None) if 'failover_timeout' not in framework: framework['failover_timeout'] = 100 return framework @property def framework_id(self): id = self._framework.get('id') return id and id.get('value') @framework_id.setter def framework_id(self, id): self._framework['id'] = dict(value=id) def _get_version(self, master): if master is not None: conn = None host, port = master.split(':', 2) port = int(port) try: conn = HTTPConnection(host, port, timeout=self._timeout) conn.request('GET', '/version') resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: return return json.loads(resp.read().decode('utf-8'))['version'] except Exception: logger.exception('Error') pass finally: if conn: conn.close() def change_master(self, master): self.version = self._get_version(master) super(MesosSchedulerDriver, self).change_master(master) self._close() def start(self): super(MesosSchedulerDriver, self).start() uri = self.master_uri if uri.startswith('zk://') or uri.startswith('zoo://'): from .detector import MasterDetector self.detector = MasterDetector(uri[uri.index('://') + 3:], self) self.detector.start() else: if ':' not in uri: uri += ':5050' self.change_master(uri) def stop(self, failover=False): with self._lock: self._failover = failover detector = self.detector self.detector = None if detector: detector.stop() super(MesosSchedulerDriver, self).stop() def _shutdown(self): if not self._failover: try: self._teardown() except Exception: logger.exception('Failed to Teardown') def _get_conn(self): if not self.connected: return None if self._conn is not None: return self._conn host, port = self.master.split(':', 2) port = int(port) self._conn = HTTPConnection(host, port, timeout=self._timeout) return self._conn def _send(self, body, path='/api/v1/scheduler', method='POST', headers={}): with self._lock: conn = self._get_conn() if conn is None: raise RuntimeError('Not connected yet') if body != '': data = json.dumps(body).encode('utf-8') headers['Content-Type'] = 'application/json' else: data = b'' stream_id = self.stream_id if stream_id: headers['Mesos-Stream-Id'] = stream_id if self._basic_credential: headers['Authorization'] = self._basic_credential try: conn.request(method, path, body=data, headers=headers) resp = conn.getresponse() except Exception: self._close() raise if resp.status < 200 or resp.status >= 300: raise RuntimeError('Failed to send request %s: %s\n%s' % (resp.status, resp.read(), data)) result = resp.read() if not result: return {} try: return json.loads(result.decode('utf-8')) except Exception: return {} def _teardown(self): if self.connected: framework_id = self.framework_id if framework_id: self._send( dict( type='TEARDOWN', framework_id=dict(value=framework_id, ), )) self._framework.pop('id', None) def acceptOffers(self, offer_ids, operations, filters=None): if not operations: return self.declineOffer(offer_ids, filters=filters) if not self.connected: return framework_id = self.framework_id assert framework_id accept = dict( offer_ids=[offer_ids] if isinstance(offer_ids, dict) else offer_ids, operations=operations, ) if filters is not None: accept['filters'] = filters body = dict( type='ACCEPT', framework_id=dict(value=framework_id, ), accept=accept, ) self._send(body) def acceptInverseOffers(self, offer_ids, filters=None): framework_id = self.framework_id assert framework_id accept_inverse_offers = dict( inverse_offer_ids=[offer_ids] if isinstance(offer_ids, dict ) else offer_ids) if filters is not None: accept_inverse_offers['filters'] = filters body = dict( type='ACCEPT_INVERSE_OFFERS', framework_id=dict(value=framework_id, ), accept_inverse_offers=accept_inverse_offers, ) self._send(body) def launchTasks(self, offer_ids, tasks, filters=None): if not tasks: return self.declineOffer(offer_ids, filters=filters) if not self.connected: return framework_id = self.framework_id assert framework_id operations = [dict( type='LAUNCH', launch=dict(task_infos=tasks), )] self.acceptOffers(offer_ids, operations, filters=filters) def declineOffer(self, offer_ids, filters=None): if not self.connected: return framework_id = self.framework_id assert framework_id decline = dict(offer_ids=[offer_ids] if isinstance(offer_ids, dict ) else offer_ids) if filters is not None: decline['filters'] = filters body = dict( type='DECLINE', framework_id=dict(value=framework_id, ), decline=decline, ) self._send(body) def declineInverseOffer(self, offer_ids, filters=None): if not self.connected: return framework_id = self.framework_id assert framework_id decline_inverse_offers = dict( inverse_offer_ids=[offer_ids] if isinstance(offer_ids, dict ) else offer_ids) if filters is not None: decline_inverse_offers['filters'] = filters body = dict( type='DECLINE_INVERSE_OFFERS', framework_id=dict(value=framework_id, ), decline_inverse_offers=decline_inverse_offers, ) self._send(body) def reviveOffers(self): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='REVIVE', framework_id=dict(value=framework_id, ), ) self._send(body) def suppressOffers(self): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='SUPPRESS', framework_id=dict(value=framework_id, ), ) self._send(body) def killTask(self, task_id): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='KILL', framework_id=dict(value=framework_id, ), kill=dict(task_id=task_id, ), ) self._send(body) def acknowledgeStatusUpdate(self, status): if self.connected and 'uuid' in status: framework_id = self.framework_id assert framework_id acknowledge = dict() acknowledge['agent_id'] = status['agent_id'] acknowledge['task_id'] = status['task_id'] acknowledge['uuid'] = status['uuid'] body = dict( type='ACKNOWLEDGE', framework_id=dict(value=framework_id, ), acknowledge=acknowledge, ) self._send(body) def acknowledgeOperationStatusUpdate(self, status): if self.connected and 'uuid' in status and 'operation_id' in status: framework_id = self.framework_id assert framework_id body = dict( type='ACKNOWLEDGE_OPERATION_STATUS', framework_id=dict(value=framework_id, ), acknowledge_operation_status=status, ) self._send(body) def reconcileTasks(self, tasks): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='RECONCILE', framework_id=dict(value=framework_id, ), reconcile=dict( tasks=[dict(task_id=task['task_id']) for task in tasks], ), ) self._send(body) def reconcileOperations(self, operations_): if not self.connected: return framework_id = self.framework_id assert framework_id operations = [] for op_ in operations_: op = dict(operation_id=op_['operation_id']) if 'agent_id' in op_: op['agent_id'] = op_['agent_id'] if 'resource_provider_id' in op_: op['resource_provider_id'] = op_['resource_provider_id'] operations.append(op) body = dict( type='RECONCILE_OPERATIONS', framework_id=dict(value=framework_id, ), reconcile_operations=dict(operations=operations, ), ) self._send(body) def sendFrameworkMessage(self, executor_id, agent_id, data): if not self.connected: return framework_id = self.framework_id assert framework_id message = dict( agent_id=agent_id, executor_id=executor_id, data=data, ) body = dict( type='MESSAGE', framework_id=dict(value=framework_id, ), message=message, ) self._send(body) def requestResources(self, requests): if not self.connected: return framework_id = self.framework_id assert framework_id body = dict( type='REQUEST', framework_id=dict(value=framework_id, ), request=dict(requests=requests, ), ) self._send(body) def onNewMasterDetectedMessage(self, data): master = None try: if isinstance(data, six.binary_type): data = data.decode('utf-8') parsed = json.loads(data) if parsed and "address" in parsed: ip = parsed["address"].get("ip") port = parsed["address"].get("port") if ip and port: master = "%s:%s" % (ip, port) except Exception: logger.exception("No JSON content, probably connecting " "to older Mesos version.") if master: self.change_master(master) def onNoMasterDetectedMessage(self): self.change_master(None) def gen_request(self): request = dict( type='SUBSCRIBE', subscribe=dict(framework_info=self.framework), ) if 'id' in self._framework: request['framework_id'] = self._framework['id'] data = json.dumps(request) _authorization = '' if self._basic_credential is not None: _authorization = 'Authorization: %s\r\n' % ( self._basic_credential, ) request = ('POST /api/v1/scheduler HTTP/1.1\r\nHost: %s\r\n' 'Content-Type: application/json\r\n' 'Accept: application/json\r\n%s' 'Connection: close\r\nContent-Length: %s\r\n\r\n%s') % ( self.master, _authorization, len(data), data) return request.encode('utf-8') def _close(self): if self._conn is not None: self._conn.close() self._conn = None def on_close(self): self._close() self.sched.disconnected(self) def on_subscribed(self, info): reregistered = (self.framework_id is not None) self.framework_id = info['framework_id']['value'] hostname, port = self.master.split(':', 2) port = int(port) master_info = dict( hostname=hostname, port=port, ) if self.version: master_info['version'] = self.version elif 'master_info' in info and 'version' in info['master_info']: master_info['version'] = info['master_info']['version'] if reregistered: self.sched.reregistered(self, self._dict_cls(master_info)) else: framework_id = dict(value=self.framework_id) self.sched.registered(self, self._dict_cls(framework_id), self._dict_cls(master_info)) def on_offers(self, event): offers = event.get('offers', []) if offers: self.sched.resourceOffers( self, [self._dict_cls(offer) for offer in offers]) version = self.version and tuple( int(n) for n in self.version.split('.')) if not (version and version >= (1, 0, 0)): self.on_inverse_offers(event) def on_inverse_offers(self, event): inverse_offers = event.get('inverse_offers', []) if inverse_offers: self.sched.inverseOffers( self, [self._dict_cls(offer) for offer in inverse_offers]) def on_rescind(self, event): offer_id = event['offer_id'] self.sched.offerRescinded(self, self._dict_cls(offer_id)) def on_rescind_inverse_offer(self, event): inverse_offer_id = event['inverse_offer_id'] self.sched.inverseOfferRescinded(self, self._dict_cls(inverse_offer_id)) def on_update(self, event): status = event['status'] self.sched.statusUpdate(self, self._dict_cls(status)) if self.implicit_acknowledgements: self.acknowledgeStatusUpdate(status) def on_update_operation_status(self, event): status = event['status'] self.sched.operationStatusUpdate(self, self._dict_cls(status)) if self.implicit_acknowledgements: self.acknowledgeOperationStatusUpdate(status) def on_message(self, message): executor_id = message['executor_id'] agent_id = message['agent_id'] data = message['data'] self.sched.frameworkMessage(self, self._dict_cls(executor_id), self._dict_cls(agent_id), data) def on_failure(self, failure): agent_id = failure['agent_id'] if 'executor_id' not in failure: self.sched.slaveLost(self, self._dict_cls(agent_id)) else: self.sched.executorLost(self, self._dict_cls(failure['executor_id']), self._dict_cls(agent_id), failure['status']) def on_error(self, event): message = event['message'] self.sched.error(self, message) def on_heartbeat(self): self.sched.processHeartBeat() def on_event(self, event): if 'type' in event: _type = event['type'].lower() if _type == 'heartbeat': self.on_heartbeat() return if _type not in event: logger.error('Missing `%s` in event %s' % (_type, event)) return event = event[_type] func_name = 'on_%s' % (_type, ) func = getattr(self, func_name, None) if func is not None: func(event) else: logger.error('Unknown type:%s, event:%s' % (_type, event)) else: logger.error('Unknown event:%s' % (event, ))
class GoogleSuggestions(): def __init__(self): self.hl = "en" self.conn = None def prepareQuery(self): #GET /complete/search?output=toolbar&client=youtube-psuggest&xml=true&ds=yt&hl=en&jsonp=self.gotSuggestions&q=s #self.prepQuerry = "/complete/search?output=toolbar&client=youtube&xml=true&ds=yt&" self.prepQuerry = "/complete/search?output=chrome&client=chrome&" if self.hl is not None: self.prepQuerry = self.prepQuerry + "hl=" + self.hl + "&" self.prepQuerry = self.prepQuerry + "jsonp=self.gotSuggestions&q=" print("[MyTube - GoogleSuggestions] prepareQuery:", self.prepQuerry) def getSuggestions(self, queryString): self.prepareQuery() if queryString != "": query = self.prepQuerry + quote(queryString) self.conn = HTTPConnection("google.com") try: self.conn = HTTPConnection("google.com") self.conn.request("GET", query, "", {"Accept-Encoding": "UTF-8"}) except (CannotSendRequest, gaierror, error): self.conn.close() print("[MyTube - GoogleSuggestions] Can not send request for suggestions") return None else: try: response = self.conn.getresponse() except BadStatusLine: self.conn.close() print("[MyTube - GoogleSuggestions] Can not get a response from google") return None else: if response.status == 200: data = response.read() header = response.getheader("Content-Type", "text/xml; charset=ISO-8859-1") charset = "ISO-8859-1" try: charset = header.split(";")[1].split("=")[1] print("[MyTube - GoogleSuggestions] Got charset %s" % charset) except: print("[MyTube - GoogleSuggestions] No charset in Header, falling back to %s" % charset) data = data.decode(charset).encode("utf-8") self.conn.close() return data else: self.conn.close() return None else: return None
def make_connection(self, server): host, port = server.server_address connection = HTTPConnection(host, port) self.addCleanup(connection.close) return connection
def test_fails(self, HTTPConnection): # coverage conn = HTTPConnection() conn.request.side_effect = socket.error rv = self.call_it('127.0.0.1', 8080, timeout=0.01, retries=2) self.assertFalse(rv)
def make_connection(self): if self.scheme == 'https': return HTTPSConnection('%s:%s' % (self.interface(), self.PORT)) else: return HTTPConnection('%s:%s' % (self.interface(), self.PORT))
class Client: """ Crossbar.io HTTP bridge client. """ def __init__(self, url, key = None, secret = None, timeout = 5, context = None): """ Create a new Crossbar.io push client. The only mandatory argument is the Push service endpoint of the Crossbar.io instance to push to. For signed pushes, provide authentication key and secret. If those are not given, unsigned pushes are performed. :param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push). :type url: str :param key: Optional key to use for signing requests. :type key: str :param secret: When using signed request, the secret corresponding to key. :type secret: str :param timeout: Timeout for requests. :type timeout: int :param context: If the HTTP bridge is running on HTTPS (that is securely over TLS), then the context provides the SSL settings the client should use (e.g. the certificate chain against which to verify the server certificate). This parameter is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext :type context: obj or None """ if six.PY2: if type(url) == str: url = six.u(url) if type(key) == str: key = six.u(key) if type(secret) == str: secret = six.u(secret) assert(type(url) == six.text_type) assert((key and secret) or (not key and not secret)) assert(key is None or type(key) == six.text_type) assert(secret is None or type(secret) == six.text_type) assert(type(timeout) == int) if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT: assert(context is None or isinstance(context, ssl.SSLContext)) self._seq = 1 self._key = key self._secret = secret self._endpoint = _parse_url(url) self._endpoint['headers'] = { "Content-type": "application/json", "User-agent": "crossbarconnect-python" } if self._endpoint['secure']: if not _HAS_SSL: raise Exception("Bridge URL is using HTTPS, but Python SSL module is missing") if _HAS_SSL_CLIENT_CONTEXT: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout = timeout, context = context) else: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout = timeout) else: self._connection = HTTPConnection(self._endpoint['host'], self._endpoint['port'], timeout = timeout) def publish(self, topic, *args, **kwargs): """ Publish an event to subscribers on specified topic via Crossbar.io HTTP bridge. The event payload (positional and keyword) can be of any type that can be serialized to JSON. If `kwargs` contains an `options` attribute, this is expected to be a dictionary with the following possible parameters: * `exclude`: A list of WAMP session IDs to exclude from receivers. * `eligible`: A list of WAMP session IDs eligible as receivers. :param topic: Topic to push to. :type topic: str :param args: Arbitrary application payload for the event (positional arguments). :type args: list :param kwargs: Arbitrary application payload for the event (keyword arguments). :type kwargs: dict :returns int -- The event publication ID assigned by the broker. """ if six.PY2 and type(topic) == str: topic = six.u(topic) assert(type(topic) == six.text_type) ## this will get filled and later serialized into HTTP/POST body ## event = { 'topic': topic } if 'options' in kwargs: event['options'] = kwargs.pop('options') assert(type(event['options']) == dict) if args: event['args'] = args if kwargs: event['kwargs'] = kwargs try: body = json.dumps(event, separators = (',',':')) if six.PY3: body = body.encode('utf8') except Exception as e: raise Exception("invalid event payload - not JSON serializable: {0}".format(e)) params = { 'timestamp': _utcnow(), 'seq': self._seq, } if self._key: ## if the request is to be signed, create extra fields and signature params['key'] = self._key params['nonce'] = random.randint(0, 9007199254740992) # HMAC[SHA256]_{secret} (key | timestamp | seq | nonce | body) => signature hm = hmac.new(self._secret.encode('utf8'), None, hashlib.sha256) hm.update(params['key'].encode('utf8')) hm.update(params['timestamp'].encode('utf8')) hm.update(u"{0}".format(params['seq']).encode('utf8')) hm.update(u"{0}".format(params['nonce']).encode('utf8')) hm.update(body) signature = base64.urlsafe_b64encode(hm.digest()) params['signature'] = signature self._seq += 1 path = "{0}?{1}".format(parse.quote(self._endpoint['path']), parse.urlencode(params)) ## now issue the HTTP/POST ## self._connection.request('POST', path, body, self._endpoint['headers']) response = self._connection.getresponse() response_body = response.read().decode() if response.status != 202: raise Exception("publication request failed {0} [{1}] - {2}".format(response.status, response.reason, response_body)) try: res = json.loads(response_body) except Exception as e: raise Exception("publication request bogus result - {0}".format(e)) return res['id']
class Client: """ Crossbar.io HTTP bridge client. """ def __init__(self, url, key=None, secret=None, timeout=5, context=None): """ Create a new Crossbar.io push client. The only mandatory argument is the Push service endpoint of the Crossbar.io instance to push to. For signed pushes, provide authentication key and secret. If those are not given, unsigned pushes are performed. :param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push). :type url: str :param key: Optional key to use for signing requests. :type key: str :param secret: When using signed request, the secret corresponding to key. :type secret: str :param timeout: Timeout for requests. :type timeout: int :param context: If the HTTP bridge is running on HTTPS (that is securely over TLS), then the context provides the SSL settings the client should use (e.g. the certificate chain against which to verify the server certificate). This parameter is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext :type context: obj or None """ if six.PY2: if type(url) == str: url = six.u(url) if type(key) == str: key = six.u(key) if type(secret) == str: secret = six.u(secret) assert (type(url) == six.text_type) assert ((key and secret) or (not key and not secret)) assert (key is None or type(key) == six.text_type) assert (secret is None or type(secret) == six.text_type) assert (type(timeout) == int) if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT: assert (context is None or isinstance(context, ssl.SSLContext)) self._seq = 1 self._key = key self._secret = secret self._endpoint = _parse_url(url) self._endpoint['headers'] = { "Content-type": "application/json", "User-agent": "crossbarconnect-python" } if self._endpoint['secure']: if not _HAS_SSL: raise Exception( "Bridge URL is using HTTPS, but Python SSL module is missing" ) if _HAS_SSL_CLIENT_CONTEXT: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout, context=context) else: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout) else: self._connection = HTTPConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout) def publish(self, topic, *args, **kwargs): """ Publish an event to subscribers on specified topic via Crossbar.io HTTP bridge. The event payload (positional and keyword) can be of any type that can be serialized to JSON. If `kwargs` contains an `options` attribute, this is expected to be a dictionary with the following possible parameters: * `exclude`: A list of WAMP session IDs to exclude from receivers. * `eligible`: A list of WAMP session IDs eligible as receivers. :param topic: Topic to push to. :type topic: str :param args: Arbitrary application payload for the event (positional arguments). :type args: list :param kwargs: Arbitrary application payload for the event (keyword arguments). :type kwargs: dict :returns int -- The event publication ID assigned by the broker. """ if six.PY2 and type(topic) == str: topic = six.u(topic) assert (type(topic) == six.text_type) ## this will get filled and later serialized into HTTP/POST body ## event = {'topic': topic} if 'options' in kwargs: event['options'] = kwargs.pop('options') assert (type(event['options']) == dict) if args: event['args'] = args if kwargs: event['kwargs'] = kwargs try: body = json.dumps(event, separators=(',', ':')) if six.PY3: body = body.encode('utf8') except Exception as e: raise Exception( "invalid event payload - not JSON serializable: {0}".format(e)) params = { 'timestamp': _utcnow(), 'seq': self._seq, } if self._key: ## if the request is to be signed, create extra fields and signature params['key'] = self._key params['nonce'] = random.randint(0, 9007199254740992) # HMAC[SHA256]_{secret} (key | timestamp | seq | nonce | body) => signature hm = hmac.new(self._secret.encode('utf8'), None, hashlib.sha256) hm.update(params['key'].encode('utf8')) hm.update(params['timestamp'].encode('utf8')) hm.update(u"{0}".format(params['seq']).encode('utf8')) hm.update(u"{0}".format(params['nonce']).encode('utf8')) hm.update(body) signature = base64.urlsafe_b64encode(hm.digest()) params['signature'] = signature self._seq += 1 path = "{0}?{1}".format(parse.quote(self._endpoint['path']), parse.urlencode(params)) ## now issue the HTTP/POST ## self._connection.request('POST', path, body, self._endpoint['headers']) response = self._connection.getresponse() response_body = response.read() if response.status != 202: raise Exception( "publication request failed {0} [{1}] - {2}".format( response.status, response.reason, response_body)) try: res = json.loads(response_body) except Exception as e: raise Exception("publication request bogus result - {0}".format(e)) return res['id']
def __init__(self, path): HTTPConnection.__init__(self, 'localhost') self.path = path
def __init__(self, *args, **kwargs): HTTPConnection.__init__(self, *args, **kwargs) self._canceled = False self.deadline = 0
def __init__(self, url, key=None, secret=None, timeout=5, context=None): """ Create a new Crossbar.io push client. The only mandatory argument is the Push service endpoint of the Crossbar.io instance to push to. For signed pushes, provide authentication key and secret. If those are not given, unsigned pushes are performed. :param url: URL of the HTTP bridge of Crossbar.io (e.g. http://example.com:8080/push). :type url: str :param key: Optional key to use for signing requests. :type key: str :param secret: When using signed request, the secret corresponding to key. :type secret: str :param timeout: Timeout for requests. :type timeout: int :param context: If the HTTP bridge is running on HTTPS (that is securely over TLS), then the context provides the SSL settings the client should use (e.g. the certificate chain against which to verify the server certificate). This parameter is only available on Python 2.7.9+ and Python 3 (otherwise the parameter is silently ignored!). See: https://docs.python.org/2/library/ssl.html#ssl.SSLContext :type context: obj or None """ if six.PY2: if type(url) == str: url = six.u(url) if type(key) == str: key = six.u(key) if type(secret) == str: secret = six.u(secret) assert (type(url) == six.text_type) assert ((key and secret) or (not key and not secret)) assert (key is None or type(key) == six.text_type) assert (secret is None or type(secret) == six.text_type) assert (type(timeout) == int) if _HAS_SSL and _HAS_SSL_CLIENT_CONTEXT: assert (context is None or isinstance(context, ssl.SSLContext)) self._seq = 1 self._key = key self._secret = secret self._endpoint = _parse_url(url) self._endpoint['headers'] = { "Content-type": "application/json", "User-agent": "crossbarconnect-python" } if self._endpoint['secure']: if not _HAS_SSL: raise Exception( "Bridge URL is using HTTPS, but Python SSL module is missing" ) if _HAS_SSL_CLIENT_CONTEXT: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout, context=context) else: self._connection = HTTPSConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout) else: self._connection = HTTPConnection(self._endpoint['host'], self._endpoint['port'], timeout=timeout)