def create_tunnel_connection(tunnel_opts: TunnelOptions,
dest_host: str,
dest_port: int,
server_name: str = None,
proxy: ProxyOptions = None):
conn = HTTPConnection(tunnel_opts.host, tunnel_opts.port)
headers = {
"Authorization":
generate_basic_header(tunnel_opts.auth_login,
tunnel_opts.auth_password),
"Client":
tunnel_opts.client.value,
"Connection":
'keep-alive',
"Server-Name":
server_name or dest_host,
"Host":
tunnel_opts.host,
"Secure":
str(int(tunnel_opts.secure)),
"HTTP2":
str(int(tunnel_opts.http2)),
}
if proxy:
headers["Proxy"] = generate_proxy_url(proxy=proxy)
conn.set_tunnel(dest_host, port=dest_port, headers=headers)
conn.connect()
return conn
def createConnection(self):
if self.debug:
conn = HTTPConnection("127.0.0.1", Connection.debugPort)
conn.set_tunnel(self.server)
else:
conn = HTTPConnection(self.server)
return conn
def createConnection(self):
if self.debug:
conn = HTTPConnection('127.0.0.1', Connection.debugPort)
conn.set_tunnel(self.server)
else:
conn = HTTPConnection(self.server)
return conn
def _do_read(self, query, raw=False):
# send query to server, return JSON
rethinker = doublethink.Rethinker(db="trough_configuration",
servers=self.rethinkdb)
healthy_databases = list(
rethinker.table('services').get_all(self.database,
index='segment').run())
healthy_databases = [
db for db in healthy_databases if db['role'] == 'trough-read' and
(rethinker.now().run() - db['last_heartbeat']).seconds < db['ttl']
]
try:
assert len(healthy_databases) > 0
except:
raise Exception('No healthy node found for segment %s' %
self.database)
url = urlparse(healthy_databases[0].get('url'))
if self.proxy:
conn = HTTPConnection(self.proxy, self.proxy_port)
conn.set_tunnel(url.netloc, url.port)
conn.sock = socks.socksocket()
conn.sock.set_proxy(self.proxy_type, self.proxy, self.proxy_port)
conn.sock.connect((url.netloc.split(":")[0], url.port))
else:
conn = HTTPConnection(url.netloc)
request_path = "%s?%s" % (url.path, url.query)
conn.request("POST", request_path, query)
response = conn.getresponse()
results = json.loads(response.read())
self._last_results = results
def http_proxy_socket(address, proxy, auth=None, headers=None):
http_con = HTTPConnection(proxy[0], proxy[1])
headers = headers if headers is not None else {}
if auth is not None:
headers['proxy-authorization'] = 'Basic ' + b64encode(('%s:%s' % (auth[0], auth[1])).encode()).decode()
http_con.set_tunnel(address[0], address[1], headers)
http_con.connect()
return http_con.sock
def getCookie():
web = HTTPConnection('localhost', 8118)
web.set_tunnel('158.69.76.135')
web.request('GET', '/level4.php')
response = web.getresponse()
cookie = response.getheader('Set-Cookie')
cookie = cookie.partition('=')[2].partition(';')[0]
web.close()
return cookie
def get_tunneled_connection(host, port, proxy):
headers = {}
if proxy.username and proxy.password:
headers['Proxy-Authorization'] = get_proxy_auth_header(proxy)
connection = HTTPConnection(proxy.host, proxy.port)
connection.set_tunnel(host, port, headers)
connection.connect()
return connection
def request(method, urlString, body = '', headers = {}, auth = None, verbose = False, https_proxy = os.getenv('https_proxy', None), timeout = 60):
url = urlparse(urlString)
if url.scheme == 'http':
conn = HTTPConnection(url.netloc, timeout = timeout)
else:
if httpRequestProps['secure'] or not hasattr(ssl, '_create_unverified_context'):
conn = HTTPSConnection(url.netloc if https_proxy is None else https_proxy, timeout = timeout)
else:
conn = HTTPSConnection(url.netloc if https_proxy is None else https_proxy, context=ssl._create_unverified_context(), timeout = timeout)
if https_proxy:
conn.set_tunnel(url.netloc)
if auth is not None:
auth = base64.b64encode(auth.encode()).decode()
headers['Authorization'] = 'Basic %s' % auth
if verbose:
print('========')
print('REQUEST:')
print('%s %s' % (method, urlString))
print('Headers sent:')
print(getPrettyJson(headers))
if body != '':
print('Body sent:')
print(body)
try:
conn.request(method, urlString, body, headers)
res = conn.getresponse()
body = ''
try:
body = res.read()
except IncompleteRead as e:
body = e.partial
# patch the read to return just the body since the normal read
# can only be done once
res.read = lambda: body
if verbose:
print('--------')
print('RESPONSE:')
print('Got response with code %s' % res.status)
print('Body received:')
print(res.read())
print('========')
return res
except socket.timeout:
return ErrorResponse(status = 500, error = 'request timed out at %d seconds' % timeout)
except Exception as e:
return ErrorResponse(status = 500, error = str(e))
def request(method, urlString, body = '', headers = {}, auth = None, verbose = False, https_proxy = os.getenv('https_proxy', None), timeout = 10):
url = urlparse(urlString)
if url.scheme == 'http':
conn = HTTPConnection(url.netloc, timeout = timeout)
else:
if httpRequestProps['secure'] or not hasattr(ssl, '_create_unverified_context'):
conn = HTTPSConnection(url.netloc if https_proxy is None else https_proxy, timeout = timeout)
else:
conn = HTTPSConnection(url.netloc if https_proxy is None else https_proxy, context=ssl._create_unverified_context(), timeout = timeout)
if https_proxy:
conn.set_tunnel(url.netloc)
if auth is not None:
auth = base64.b64encode(auth.encode()).decode()
headers['Authorization'] = 'Basic %s' % auth
if verbose:
print('========')
print('REQUEST:')
print('%s %s' % (method, urlString))
print('Headers sent:')
print(getPrettyJson(headers))
if body != '':
print('Body sent:')
print(body)
try:
conn.request(method, urlString, body, headers)
res = conn.getresponse()
body = ''
try:
body = res.read()
except IncompleteRead as e:
body = e.partial
# patch the read to return just the body since the normal read
# can only be done once
res.read = lambda: body
if verbose:
print('--------')
print('RESPONSE:')
print('Got response with code %s' % res.status)
print('Body received:')
print(res.read())
print('========')
return res
except socket.timeout:
return ErrorResponse(status = 500, error = 'request timed out at %d seconds' % timeout)
except Exception as e:
return ErrorResponse(status = 500, error = str(e))
def post(cookie):
web = HTTPConnection('localhost', 8118)
web.set_tunnel('158.69.76.135')
web.connect()
web.putrequest('POST', '/level4.php')
web.putheader('Cookie', 'HoldTheDoor=' + cookie)
web.putheader('Content-Type', 'application/x-www-form-urlencoded')
web.putheader('Referer', 'http://158.69.76.135/level4.php')
body = b'id=701&holdthedoor=Submit&key=' + cookie.encode('ASCII')
web.putheader('Content-Length', str(len(body)))
web.endheaders()
web.send(body)
response = web.getresponse()
web.close()
def RunCmd(objCommand):
host_proxy = "connect2.virtual.uniandes.edu.co"
port_proxy = 443
host_server = "bigdata-cluster1-ambari.virtual.uniandes.edu.co"
port_server = 22
user_server = "bigdata09"
pass_server = "Rojo2020"
http_con = HTTPConnection(host_proxy, port_proxy)
http_con.set_tunnel(host_server, port_server)
http_con.connect()
sock = http_con.sock
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host_server,
username=user_server,
password=pass_server,
sock=sock)
ssh.exec_command(objCommand)
ssh.close()
http_con.close()
def DownloadFile(objRemoteFile, objLocalFile):
host_proxy = "connect2.virtual.uniandes.edu.co"
port_proxy = 443
host_server = "bigdata-cluster1-ambari.virtual.uniandes.edu.co"
port_server = 22
user_server = "bigdata09"
pass_server = "Rojo2020"
http_con = HTTPConnection(host_proxy, port_proxy)
http_con.set_tunnel(host_server, port_server)
http_con.connect()
sock = http_con.sock
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host_server,
username=user_server,
password=pass_server,
sock=sock)
sftp = ssh.open_sftp()
sftp.get(objRemoteFile, objLocalFile)
sftp.close()
ssh.close()
http_con.close()
def make_connection(self, host):
connection = HTTPConnection(*self.proxy)
connection.set_tunnel(host, headers=self.proxy_headers)
self._connection = host, connection
return connection
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(self,
host='api.dynect.net',
port=443,
ssl=True,
api_version="current",
proxy_host=None,
proxy_port=None):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
# Add support for proxy connections
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.use_proxy = False
self.proxied_root = None
if proxy_host is not None and proxy_port is not None:
self.use_proxy = True
scheme = 'https' if self.ssl else 'http'
self.proxied_root = "{}://{}".format(scheme, self.host)
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def _connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def _proxy_connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.proxy_host, self.proxy_port)
else:
msg = "Establishing SSL connection to %s:%s\n" % (self.proxy_host,
self.proxy_port)
self._debug(msg)
self._conn = HTTPConnection(self.proxy_host, self.proxy_port)
self._conn.set_tunnel(self.host, self.port)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.use_proxy:
self._proxy_connect()
else:
self._connect()
def execute(self, uri, method, args=None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods))
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in list(headers.items()):
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
def request(method,
urlString,
body='',
headers={},
auth=None,
verbose=False,
https_proxy=os.getenv('https_proxy', None),
timeout=60):
url = urlparse(urlString)
if url.scheme == 'http':
conn = HTTPConnection(url.netloc, timeout=timeout)
else:
if httpRequestProps['secure'] or not hasattr(
ssl, '_create_unverified_context'):
conn = HTTPSConnection(
url.netloc if https_proxy is None else https_proxy,
timeout=timeout)
else:
conn = HTTPSConnection(
url.netloc if https_proxy is None else https_proxy,
context=ssl._create_unverified_context(),
timeout=timeout)
if https_proxy:
conn.set_tunnel(url.netloc)
if auth is not None:
auth = base64.b64encode(auth.encode()).decode()
headers['Authorization'] = 'Basic %s' % auth
if verbose:
print('========')
print('REQUEST:')
print('%s %s' % (method, urlString))
print('Headers sent:')
print(getPrettyJson(headers))
if body != '':
print('Body sent:')
print(body)
try:
# Add random network delay
time.sleep(random.expovariate(40))
conn.request(method, urlString, body, headers)
# read reuse buffer
global rb
body_store = body
if USE_REUSE_BUFFER:
l.acquire()
matched_item = rb.loc[(rb['Url'] == urlString)
& (rb['Body'] == body_store)]
l.release()
if len(matched_item.index) > 0:
# print("Find Matched!")
if len(matched_item.index) > 1:
raise Exception("Two matched items in edge reuse buffer!!")
res = matched_item["Output"]
t_end = time.time()
global hit_count
hit_count += 1
if hit_count % 20 == 0:
print("Buffer Usage: " + str(rb.memory_usage().sum()) +
"/" + str(MAX_BUFFER_SIZE))
print("Hit Times: " + str(hit_count))
return res
# Get response
res = conn.getresponse()
body = ''
try:
body = res.read()
except IncompleteRead as e:
body = e.partial
# patch the read to return just the body since the normal read
# can only be done once
res.read = lambda: body
if verbose:
print('--------')
print('RESPONSE:')
print('Got response with code %s' % res.status)
print('Body received:')
print(res.read())
print('========')
# store Reuse Buffer
if USE_REUSE_BUFFER:
l.acquire()
# Need to judge whether there is matched item again before writing
matched_item = rb.loc[(rb['Url'] == urlString)
& (rb['Body'] == body_store)]
if len(matched_item.index) > 0:
l.release()
return res.read().decode()
rb = rb.append(
{
'Url': urlString,
'Body': body_store,
'Output': res.read().decode()
},
ignore_index=True)
# Need to judge whether to kick unused items out
# If the buffer is full, kick the first a few elements out of the buffer
if (not rb.empty) and rb.memory_usage().sum() > MAX_BUFFER_SIZE:
idx = 0
while (not rb.empty
) and rb[idx:].memory_usage().sum() > MAX_BUFFER_SIZE:
idx += 1
rb = rb[idx:]
l.release()
return res.read().decode()
except socket.timeout:
return ErrorResponse(status=500,
error='request timed out at %d seconds' % timeout)
except Exception as e:
return ErrorResponse(status=500, error=str(e))
class OSMNet():
"""Connects to OpenStreetMap, downloads and converts specified network."""
def __init__(self, api="www.overpass-api.de/api/interpreter"):
"""Initialize OSM downloader
Params
------
api : str (default: "www.overpass-api.de/api/interpreter")
API URL to use for download
"""
# parse url
if "http" in api:
url = urlparse(api)
else:
url = urlparse("https://" + api)
self.path = url.path
# proxy check
if os.environ.get("https_proxy") is not None:
headers = {}
proxy_url = urlparse(os.environ.get("https_proxy"))
if proxy_url.username and proxy_url.password:
auth = '%s:%s' % (proxy_url.username, proxy_url.password)
headers['Proxy-Authorization'] = 'Basic ' + base64.b64encode(
auth)
self.conn = HTTPSConnection(proxy_url.hostname, proxy_url.port)
self.conn.set_tunnel(url.hostname, 443, headers)
else:
# HTTP vs HTTPS check
if url.scheme == "https":
self.conn = HTTPSConnection(url.hostname, url.port)
else:
self.conn = HTTPConnection(url.hostname, url.port)
def download(self, north, south, west, east, osm_file):
"""Get the OSM network for specified coords and store to file.
Params
------
north, south, west, east : float
Geo coordinates of input area bounding box
osm_file : str
Output OSM network file name/path
"""
self.conn.request(
"POST", "/" + self.path, """
<osm-script timeout="240" element-limit="1073741824">
<union>
<bbox-query n="%s" s="%s" w="%s" e="%s"/>
<recurse type="node-relation" into="rels"/>
<recurse type="node-way"/>
<recurse type="way-relation"/>
</union>
<union>
<item/>
<recurse type="way-node"/>
</union>
<print mode="body"/>
</osm-script>""" % (north, south, west, east))
print("Downloading map data")
response = self.conn.getresponse()
print(response.status, response.reason)
if response.status == 200:
out = open(os.path.join(os.getcwd(), osm_file), "wb")
out.write(response.read())
out.close()
def convert(self, osm_file, net_file):
"""Convert OSM network to SUMO format using netconvert.
Params
------
osm_file : str
OSM network file name/path
net_file : str
Output SUMO network file name/path
"""
# get binary
netconvert = sumolib.checkBinary('netconvert')
# additional options
netconvertOpts = [netconvert]
netconvertOpts += ['--sidewalks.guess', '--crossings.guess']
# input and output files
netconvertOpts += ['--osm-files', osm_file]
netconvertOpts += ['--output-file', net_file]
return subprocess.call(netconvertOpts)
def get(self, north, south, west, east, net_file):
"""Get OSM network and convert to SUMO format.
Params
------
north, south, west, east : float
Coordinates of input bounding box
net_file : str
Output SUMO network file name/path
"""
tmp_file = "tmp.osm.xml"
self.download(north, south, west, east, tmp_file)
success = self.convert(tmp_file, net_file)
os.remove(tmp_file)
return success
