def create_tunnel_connection(tunnel_opts: TunnelOptions, dest_host: str, dest_port: int, server_name: str = None, proxy: ProxyOptions = None): conn = HTTPConnection(tunnel_opts.host, tunnel_opts.port) headers = { "Authorization": generate_basic_header(tunnel_opts.auth_login, tunnel_opts.auth_password), "Client": tunnel_opts.client.value, "Connection": 'keep-alive', "Server-Name": server_name or dest_host, "Host": tunnel_opts.host, "Secure": str(int(tunnel_opts.secure)), "HTTP2": str(int(tunnel_opts.http2)), } if proxy: headers["Proxy"] = generate_proxy_url(proxy=proxy) conn.set_tunnel(dest_host, port=dest_port, headers=headers) conn.connect() return conn
def createConnection(self): if self.debug: conn = HTTPConnection("127.0.0.1", Connection.debugPort) conn.set_tunnel(self.server) else: conn = HTTPConnection(self.server) return conn
def createConnection(self): if self.debug: conn = HTTPConnection('127.0.0.1', Connection.debugPort) conn.set_tunnel(self.server) else: conn = HTTPConnection(self.server) return conn
def _do_read(self, query, raw=False): # send query to server, return JSON rethinker = doublethink.Rethinker(db="trough_configuration", servers=self.rethinkdb) healthy_databases = list( rethinker.table('services').get_all(self.database, index='segment').run()) healthy_databases = [ db for db in healthy_databases if db['role'] == 'trough-read' and (rethinker.now().run() - db['last_heartbeat']).seconds < db['ttl'] ] try: assert len(healthy_databases) > 0 except: raise Exception('No healthy node found for segment %s' % self.database) url = urlparse(healthy_databases[0].get('url')) if self.proxy: conn = HTTPConnection(self.proxy, self.proxy_port) conn.set_tunnel(url.netloc, url.port) conn.sock = socks.socksocket() conn.sock.set_proxy(self.proxy_type, self.proxy, self.proxy_port) conn.sock.connect((url.netloc.split(":")[0], url.port)) else: conn = HTTPConnection(url.netloc) request_path = "%s?%s" % (url.path, url.query) conn.request("POST", request_path, query) response = conn.getresponse() results = json.loads(response.read()) self._last_results = results
def http_proxy_socket(address, proxy, auth=None, headers=None): http_con = HTTPConnection(proxy[0], proxy[1]) headers = headers if headers is not None else {} if auth is not None: headers['proxy-authorization'] = 'Basic ' + b64encode(('%s:%s' % (auth[0], auth[1])).encode()).decode() http_con.set_tunnel(address[0], address[1], headers) http_con.connect() return http_con.sock
def getCookie(): web = HTTPConnection('localhost', 8118) web.set_tunnel('158.69.76.135') web.request('GET', '/level4.php') response = web.getresponse() cookie = response.getheader('Set-Cookie') cookie = cookie.partition('=')[2].partition(';')[0] web.close() return cookie
def get_tunneled_connection(host, port, proxy): headers = {} if proxy.username and proxy.password: headers['Proxy-Authorization'] = get_proxy_auth_header(proxy) connection = HTTPConnection(proxy.host, proxy.port) connection.set_tunnel(host, port, headers) connection.connect() return connection
def request(method, urlString, body = '', headers = {}, auth = None, verbose = False, https_proxy = os.getenv('https_proxy', None), timeout = 60): url = urlparse(urlString) if url.scheme == 'http': conn = HTTPConnection(url.netloc, timeout = timeout) else: if httpRequestProps['secure'] or not hasattr(ssl, '_create_unverified_context'): conn = HTTPSConnection(url.netloc if https_proxy is None else https_proxy, timeout = timeout) else: conn = HTTPSConnection(url.netloc if https_proxy is None else https_proxy, context=ssl._create_unverified_context(), timeout = timeout) if https_proxy: conn.set_tunnel(url.netloc) if auth is not None: auth = base64.b64encode(auth.encode()).decode() headers['Authorization'] = 'Basic %s' % auth if verbose: print('========') print('REQUEST:') print('%s %s' % (method, urlString)) print('Headers sent:') print(getPrettyJson(headers)) if body != '': print('Body sent:') print(body) try: conn.request(method, urlString, body, headers) res = conn.getresponse() body = '' try: body = res.read() except IncompleteRead as e: body = e.partial # patch the read to return just the body since the normal read # can only be done once res.read = lambda: body if verbose: print('--------') print('RESPONSE:') print('Got response with code %s' % res.status) print('Body received:') print(res.read()) print('========') return res except socket.timeout: return ErrorResponse(status = 500, error = 'request timed out at %d seconds' % timeout) except Exception as e: return ErrorResponse(status = 500, error = str(e))
def request(method, urlString, body = '', headers = {}, auth = None, verbose = False, https_proxy = os.getenv('https_proxy', None), timeout = 10): url = urlparse(urlString) if url.scheme == 'http': conn = HTTPConnection(url.netloc, timeout = timeout) else: if httpRequestProps['secure'] or not hasattr(ssl, '_create_unverified_context'): conn = HTTPSConnection(url.netloc if https_proxy is None else https_proxy, timeout = timeout) else: conn = HTTPSConnection(url.netloc if https_proxy is None else https_proxy, context=ssl._create_unverified_context(), timeout = timeout) if https_proxy: conn.set_tunnel(url.netloc) if auth is not None: auth = base64.b64encode(auth.encode()).decode() headers['Authorization'] = 'Basic %s' % auth if verbose: print('========') print('REQUEST:') print('%s %s' % (method, urlString)) print('Headers sent:') print(getPrettyJson(headers)) if body != '': print('Body sent:') print(body) try: conn.request(method, urlString, body, headers) res = conn.getresponse() body = '' try: body = res.read() except IncompleteRead as e: body = e.partial # patch the read to return just the body since the normal read # can only be done once res.read = lambda: body if verbose: print('--------') print('RESPONSE:') print('Got response with code %s' % res.status) print('Body received:') print(res.read()) print('========') return res except socket.timeout: return ErrorResponse(status = 500, error = 'request timed out at %d seconds' % timeout) except Exception as e: return ErrorResponse(status = 500, error = str(e))
def post(cookie): web = HTTPConnection('localhost', 8118) web.set_tunnel('158.69.76.135') web.connect() web.putrequest('POST', '/level4.php') web.putheader('Cookie', 'HoldTheDoor=' + cookie) web.putheader('Content-Type', 'application/x-www-form-urlencoded') web.putheader('Referer', 'http://158.69.76.135/level4.php') body = b'id=701&holdthedoor=Submit&key=' + cookie.encode('ASCII') web.putheader('Content-Length', str(len(body))) web.endheaders() web.send(body) response = web.getresponse() web.close()
def RunCmd(objCommand): host_proxy = "connect2.virtual.uniandes.edu.co" port_proxy = 443 host_server = "bigdata-cluster1-ambari.virtual.uniandes.edu.co" port_server = 22 user_server = "bigdata09" pass_server = "Rojo2020" http_con = HTTPConnection(host_proxy, port_proxy) http_con.set_tunnel(host_server, port_server) http_con.connect() sock = http_con.sock ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=host_server, username=user_server, password=pass_server, sock=sock) ssh.exec_command(objCommand) ssh.close() http_con.close()
def DownloadFile(objRemoteFile, objLocalFile): host_proxy = "connect2.virtual.uniandes.edu.co" port_proxy = 443 host_server = "bigdata-cluster1-ambari.virtual.uniandes.edu.co" port_server = 22 user_server = "bigdata09" pass_server = "Rojo2020" http_con = HTTPConnection(host_proxy, port_proxy) http_con.set_tunnel(host_server, port_server) http_con.connect() sock = http_con.sock ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=host_server, username=user_server, password=pass_server, sock=sock) sftp = ssh.open_sftp() sftp.get(objRemoteFile, objLocalFile) sftp.close() ssh.close() http_con.close()
def make_connection(self, host): connection = HTTPConnection(*self.proxy) connection.set_tunnel(host, headers=self.proxy_headers) self._connection = host, connection return connection
class DynectRest(object): """ A class for interacting with the Dynect Managed DNS REST API. @ivar host: The host to connect to (defaults to api.dynect.net) @type host: C{str} @ivar port: The port to connect to (defaults to 443) @type port: C{int} @ivar ssl: A boolean indicating whether or not to use SSL encryption (defaults to True) @type ssl: C{bool} @ivar poll_incomplete: A boolean indicating whether we should continue to poll for a result if a job comes back as incomplete (defaults to True) @type poll_incomplete: C{bool} @ivar api_version: The version of the API to request (defaults to "current") @type api_version: C{str} """ def __init__(self, host='api.dynect.net', port=443, ssl=True, api_version="current", proxy_host=None, proxy_port=None): """ Basic initializer method @param host: The host to connect to @type host: C{str} @param port: The port to connect to @type port: C{int} @param ssl: A boolean indicating whether or not to use SSL encryption @type ssl: C{bool} """ self.host = host self.port = port self.ssl = ssl # Continue polling for response if a job comes back as incomplete? self.poll_incomplete = True self.verbose = False self.api_version = api_version self.content_type = "application/json" self._token = None self._conn = None self._last_response = None self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT')) # Add support for proxy connections self.proxy_host = proxy_host self.proxy_port = proxy_port self.use_proxy = False self.proxied_root = None if proxy_host is not None and proxy_port is not None: self.use_proxy = True scheme = 'https' if self.ssl else 'http' self.proxied_root = "{}://{}".format(scheme, self.host) def _debug(self, msg): """ Debug output. """ if self.verbose: sys.stderr.write(msg) def _connect(self): if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % (self.host, self.port) self._debug(msg) self._conn = HTTPSConnection(self.host, self.port) else: msg = "Establishing unencrypted connection to %s:%s\n" % ( self.host, self.port) self._debug(msg) self._conn = HTTPConnection(self.host, self.port) def _proxy_connect(self): if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % (self.host, self.port) self._debug(msg) self._conn = HTTPSConnection(self.proxy_host, self.proxy_port) else: msg = "Establishing SSL connection to %s:%s\n" % (self.proxy_host, self.proxy_port) self._debug(msg) self._conn = HTTPConnection(self.proxy_host, self.proxy_port) self._conn.set_tunnel(self.host, self.port) def connect(self): """ Establishes a connection to the REST API server as defined by the host, port and ssl instance variables """ if self._token: self._debug("Forcing logout from old session.\n") orig_value = self.poll_incomplete self.poll_incomplete = False self.execute('/REST/Session', 'DELETE') self.poll_incomplete = orig_value self._token = None self._conn = None if self.use_proxy: self._proxy_connect() else: self._connect() def execute(self, uri, method, args=None): """ Execute a commands against the rest server @param uri: The uri of the resource to access. /REST/ will be prepended if it is not at the beginning of the uri. @type uri: C{str} @param method: One of 'DELETE', 'GET', 'POST', or 'PUT' @type method: C{str} @param args: Any arguments to be sent as a part of the request @type args: C{dict} """ if self._conn == None: self._debug("No established connection\n") self.connect() # Make sure the command is prefixed by '/REST/' if not uri.startswith('/'): uri = '/' + uri if not uri.startswith('/REST'): uri = '/REST' + uri # Make sure the method is valid if method.upper() not in self._valid_methods: msg = "%s is not a valid HTTP method. Please use one of %s" % ( method, ", ".join(self._valid_methods)) raise ValueError(msg) # Prepare arguments if args is None: args = {} args = self.format_arguments(args) self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args)) # Send the command and deal with results self.send_command(uri, method, args) # Deal with the results response = self._conn.getresponse() body = response.read() self._last_response = response if self.poll_incomplete: response, body = self.poll_response(response, body) self._last_response = response if sys.version_info[0] == 2: ret_val = json.loads(body) elif sys.version_info[0] == 3: ret_val = json.loads(body.decode('UTF-8')) self._meta_update(uri, method, ret_val) return ret_val def _meta_update(self, uri, method, results): """ Private method, not intended for use outside the class """ # If we had a successful log in, update the token if uri.startswith('/REST/Session') and method == 'POST': if results['status'] == 'success': self._token = results['data']['token'] # Otherwise, if it's a successful logout, blank the token if uri.startswith('/REST/Session') and method == 'DELETE': if results['status'] == 'success': self._token = None def poll_response(self, response, body): """ Looks at a response from a REST command, and while indicates that the job is incomplete, poll for response """ while response.status == 307: time.sleep(1) uri = response.getheader('Location') self._debug("Polling %s\n" % uri) self.send_command(uri, "GET", '') response = self._conn.getresponse() body = response.read() return response, body def send_command(self, uri, method, args): """ Responsible for packaging up the API request and sending it to the server over the established connection @param uri: The uri of the resource to interact with @type uri: C{str} @param method: The HTTP method to use @type method: C{str} @param args: Encoded arguments to send to the server @type args: C{str} """ if '%' not in uri: uri = pathname2url(uri) self._conn.putrequest(method, uri) # Build headers headers = { 'Content-Type': self.content_type, 'API-Version': self.api_version, } if self._token is not None: headers['Auth-Token'] = self._token for key, val in list(headers.items()): self._conn.putheader(key, val) # Now the arguments self._conn.putheader('Content-length', '%d' % len(args)) self._conn.endheaders() if sys.version_info[0] == 2: self._conn.send(args) elif sys.version_info[0] == 3: self._conn.send(bytes(args, 'UTF-8')) def format_arguments(self, args): """ Converts the argument dictionary to the format needed to transmit the REST request. @param args: Arguments to be passed to the REST call @type args: C{dict} @return: The encoded string to send in the REST request body @rtype: C{str} """ args = json.dumps(args) return args
def request(method, urlString, body='', headers={}, auth=None, verbose=False, https_proxy=os.getenv('https_proxy', None), timeout=60): url = urlparse(urlString) if url.scheme == 'http': conn = HTTPConnection(url.netloc, timeout=timeout) else: if httpRequestProps['secure'] or not hasattr( ssl, '_create_unverified_context'): conn = HTTPSConnection( url.netloc if https_proxy is None else https_proxy, timeout=timeout) else: conn = HTTPSConnection( url.netloc if https_proxy is None else https_proxy, context=ssl._create_unverified_context(), timeout=timeout) if https_proxy: conn.set_tunnel(url.netloc) if auth is not None: auth = base64.b64encode(auth.encode()).decode() headers['Authorization'] = 'Basic %s' % auth if verbose: print('========') print('REQUEST:') print('%s %s' % (method, urlString)) print('Headers sent:') print(getPrettyJson(headers)) if body != '': print('Body sent:') print(body) try: # Add random network delay time.sleep(random.expovariate(40)) conn.request(method, urlString, body, headers) # read reuse buffer global rb body_store = body if USE_REUSE_BUFFER: l.acquire() matched_item = rb.loc[(rb['Url'] == urlString) & (rb['Body'] == body_store)] l.release() if len(matched_item.index) > 0: # print("Find Matched!") if len(matched_item.index) > 1: raise Exception("Two matched items in edge reuse buffer!!") res = matched_item["Output"] t_end = time.time() global hit_count hit_count += 1 if hit_count % 20 == 0: print("Buffer Usage: " + str(rb.memory_usage().sum()) + "/" + str(MAX_BUFFER_SIZE)) print("Hit Times: " + str(hit_count)) return res # Get response res = conn.getresponse() body = '' try: body = res.read() except IncompleteRead as e: body = e.partial # patch the read to return just the body since the normal read # can only be done once res.read = lambda: body if verbose: print('--------') print('RESPONSE:') print('Got response with code %s' % res.status) print('Body received:') print(res.read()) print('========') # store Reuse Buffer if USE_REUSE_BUFFER: l.acquire() # Need to judge whether there is matched item again before writing matched_item = rb.loc[(rb['Url'] == urlString) & (rb['Body'] == body_store)] if len(matched_item.index) > 0: l.release() return res.read().decode() rb = rb.append( { 'Url': urlString, 'Body': body_store, 'Output': res.read().decode() }, ignore_index=True) # Need to judge whether to kick unused items out # If the buffer is full, kick the first a few elements out of the buffer if (not rb.empty) and rb.memory_usage().sum() > MAX_BUFFER_SIZE: idx = 0 while (not rb.empty ) and rb[idx:].memory_usage().sum() > MAX_BUFFER_SIZE: idx += 1 rb = rb[idx:] l.release() return res.read().decode() except socket.timeout: return ErrorResponse(status=500, error='request timed out at %d seconds' % timeout) except Exception as e: return ErrorResponse(status=500, error=str(e))
class OSMNet(): """Connects to OpenStreetMap, downloads and converts specified network.""" def __init__(self, api="www.overpass-api.de/api/interpreter"): """Initialize OSM downloader Params ------ api : str (default: "www.overpass-api.de/api/interpreter") API URL to use for download """ # parse url if "http" in api: url = urlparse(api) else: url = urlparse("https://" + api) self.path = url.path # proxy check if os.environ.get("https_proxy") is not None: headers = {} proxy_url = urlparse(os.environ.get("https_proxy")) if proxy_url.username and proxy_url.password: auth = '%s:%s' % (proxy_url.username, proxy_url.password) headers['Proxy-Authorization'] = 'Basic ' + base64.b64encode( auth) self.conn = HTTPSConnection(proxy_url.hostname, proxy_url.port) self.conn.set_tunnel(url.hostname, 443, headers) else: # HTTP vs HTTPS check if url.scheme == "https": self.conn = HTTPSConnection(url.hostname, url.port) else: self.conn = HTTPConnection(url.hostname, url.port) def download(self, north, south, west, east, osm_file): """Get the OSM network for specified coords and store to file. Params ------ north, south, west, east : float Geo coordinates of input area bounding box osm_file : str Output OSM network file name/path """ self.conn.request( "POST", "/" + self.path, """ <osm-script timeout="240" element-limit="1073741824"> <union> <bbox-query n="%s" s="%s" w="%s" e="%s"/> <recurse type="node-relation" into="rels"/> <recurse type="node-way"/> <recurse type="way-relation"/> </union> <union> <item/> <recurse type="way-node"/> </union> <print mode="body"/> </osm-script>""" % (north, south, west, east)) print("Downloading map data") response = self.conn.getresponse() print(response.status, response.reason) if response.status == 200: out = open(os.path.join(os.getcwd(), osm_file), "wb") out.write(response.read()) out.close() def convert(self, osm_file, net_file): """Convert OSM network to SUMO format using netconvert. Params ------ osm_file : str OSM network file name/path net_file : str Output SUMO network file name/path """ # get binary netconvert = sumolib.checkBinary('netconvert') # additional options netconvertOpts = [netconvert] netconvertOpts += ['--sidewalks.guess', '--crossings.guess'] # input and output files netconvertOpts += ['--osm-files', osm_file] netconvertOpts += ['--output-file', net_file] return subprocess.call(netconvertOpts) def get(self, north, south, west, east, net_file): """Get OSM network and convert to SUMO format. Params ------ north, south, west, east : float Coordinates of input bounding box net_file : str Output SUMO network file name/path """ tmp_file = "tmp.osm.xml" self.download(north, south, west, east, tmp_file) success = self.convert(tmp_file, net_file) os.remove(tmp_file) return success