def query_host(world):
u = urlparse(world['status'])
c = HTTPConnection(u.netloc, 80)
c.putrequest("GET", u.path + '?' + u.query)
c.putheader("Content-Type", "text/xml; charset=utf-8")
c.endheaders()
r = c.getresponse()
if r.getcode() is not 200:
raise RuntimeError("Failed to query information about the server.")
rdata = r.read().decode("utf-8")
if len(rdata) is 0:
print("The given server appears to be down.")
exit(0)
xml = ElementTree.fromstring(rdata)
loginserver = xml.find("loginservers").text
loginservers = loginserver.split(';')
worldqueue = xml.find("queueurls").text
worldqueues = worldqueue.split(';')
world['host'] = world['login'] = loginservers[0]
world['queue'] = worldqueues[0]
return world
def _sendChunked(self, conn:client.HTTPConnection):
conn.putrequest(self.method, self.selector,
skip_accept_encoding=True)
for header, value in self._headers.items():
conn.putheader(header, value)
conn.endheaders()
def send_line(line):
chunk = line.rstrip()
if chunk:
if hasattr(chunk, 'encode'):
# http.client API and HTTP spec conform encoding!
chunk = chunk.encode('iso-8859-1')
chunk = (b'%x\r\n' % len(chunk)) + chunk + b'\r\n'
conn.send(chunk)
if hasattr(self.body, 'readline'):
line = self.body.readline()
while line:
send_line(line)
line = self.body.readline()
else:
for line in self.body.splitlines():
send_line(line)
def send_encoded(self, url, data):
url2 = urlparse(url)
host = url2.netloc
path = url2.path or '/'
query = '?' + url2.query if url2.query else ''
request = path + query
data2 = '&'.join(
(quote(k) + '=' + quote_from_bytes(str(v).encode('utf8'))
for k, v in data.items())).encode('ascii')
try:
http = HTTPConnection(host)
http.putrequest('POST', request)
http.putheader('Content-Type', 'application/x-www-form-urlencoded')
http.putheader('User-Agent', 'Fluxid MOC Scrobbler 0.2')
http.putheader('Content-Length', str(len(data2)))
http.endheaders()
http.send(data2)
response = http.getresponse().read().decode('utf8').upper().strip()
except Exception as e:
raise HardErrorException(str(e))
if response == 'BADSESSION':
raise BadSessionException
elif response.startswith('FAILED'):
raise FailedException(
response.split(' ', 1)[1].strip() + (' POST = [%r]' % data2))
def urlopen(url, svprev, formdata):
ua = "SPlayer Build %d" % svprev
#prepare data
#generate a random boundary
boundary = "----------------------------" + "%x" % random.getrandbits(48)
data = []
for item in formdata:
data.append("--" + boundary +
"\r\nContent-Disposition: form-data; name=\"" + item[0] +
"\"\r\n\r\n" + item[1] + "\r\n")
data.append("--" + boundary + "--\r\n")
data = "".join(data)
cl = str(len(data))
r = urlparse(url)
h = HTTPConnection(r.hostname)
h.connect()
h.putrequest("POST", r.path, skip_host=True, skip_accept_encoding=True)
h.putheader("User-Agent", ua)
h.putheader("Host", r.hostname)
h.putheader("Accept", "*/*")
h.putheader("Content-Length", cl)
h.putheader("Expect", "100-continue")
h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary)
h.endheaders()
h.send(data)
resp = h.getresponse()
if resp.status != OK:
raise Exception("HTTP response " + str(resp.status) + ": " +
resp.reason)
return resp
def query_datacenters(game = "DDO", datacenterurl = "http://gls.ddo.com/GLS.DataCenterServer/Service.asmx"):
url = urlparse(datacenterurl)
soaprequest = """<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<soap:Body>
<GetDatacenters xmlns="http://www.turbine.com/SE/GLS">
<game>%s</game>
</GetDatacenters>
</soap:Body>
</soap:Envelope>
""" % (game)
c = HTTPConnection(url.netloc, 80)
c.putrequest("POST", url.path)
c.putheader("Content-Type", "text/xml; charset=utf-8")
c.putheader("SOAPAction", "http://www.turbine.com/SE/GLS/GetDatacenters")
c.putheader("Content-Length", str(len(soaprequest)))
c.endheaders()
c.send(bytes(soaprequest, "utf-8"))
r = c.getresponse()
if r.getcode() is not 200:
raise LoginError('Failed to query data center for information.')
xml = _getxmlresponse(r)
dcs = []
datacenters = xml.findall('Body/GetDatacentersResponse/GetDatacentersResult/*')
for dc in datacenters:
datacenter = DataCenter()
datacenter._parse_xml(dc)
dcs.append(datacenter)
return dcs
def _query_details(self):
try:
u = urlparse(self._statusurl)
c = HTTPConnection(u.netloc, u.port)
c.putrequest("GET", u.path + '?' + u.query)
c.putheader("Content-Type", "text/xml; charset=utf-8")
c.endheaders()
r = c.getresponse()
if r.getcode() is not 200:
raise LoginError("Failed to query information about the server.")
xml = _getxmlresponse(r)
loginserver = xml.find("loginservers").text
self._loginservers = loginserver.split(';')
if len(self._loginservers) == 0:
raise LoginError('World provided no login server or servers.')
worldqueue = xml.find("queueurls").text
self._worldqueues = worldqueue.split(';')
if len(self._worldqueues) == 0:
raise LoginError('World provided no queue or queues.')
self._down = False
except xml.etree.ElementTree.ParseError as pe:
# XML error means wrong or no reply: Server is down.
self._down = True
except: # Rethrow other exceptions
raise
def send_encoded(self, url, data):
url2 = urlparse(url)
host = url2.netloc
path = url2.path or '/'
query = '?' + url2.query if url2.query else ''
request = path + query
data2 = '&'.join((
quote(k) + '=' + quote_from_bytes(str(v).encode('utf8'))
for k, v in data.items()
)).encode('ascii')
try:
http = HTTPConnection(host)
http.putrequest('POST', request)
http.putheader('Content-Type', 'application/x-www-form-urlencoded')
http.putheader('User-Agent', 'Fluxid MOC Scrobbler 0.2')
http.putheader('Content-Length', str(len(data2)))
http.endheaders()
http.send(data2)
response = http.getresponse().read().decode('utf8').upper().strip()
except Exception as e:
raise HardErrorException(str(e))
if response == 'BADSESSION':
raise BadSessionException
elif response.startswith('FAILED'):
raise FailedException(response.split(' ', 1)[1].strip() + (' POST = [%r]' % data2))
def websocket_handshake(
self,
test_app,
path="/",
params=None,
headers=None,
subprotocols=None,
timeout=1,
):
"""
Runs a WebSocket handshake negotiation and returns the raw socket
object & the selected subprotocol.
You'll need to inject an accept or reject message before this
to let it complete.
"""
# Send it the request. We have to do this the long way to allow
# duplicate headers.
conn = HTTPConnection(test_app.host, test_app.port, timeout=timeout)
if params:
path += "?" + parse.urlencode(params, doseq=True)
conn.putrequest("GET", path, skip_accept_encoding=True, skip_host=True)
# Do WebSocket handshake headers + any other headers
if headers is None:
headers = []
headers.extend([
(b"Host", b"example.com"),
(b"Upgrade", b"websocket"),
(b"Connection", b"Upgrade"),
(b"Sec-WebSocket-Key", b"x3JJHMbDL1EzLkh9GBhXDw=="),
(b"Sec-WebSocket-Version", b"13"),
(b"Origin", b"http://example.com"),
])
if subprotocols:
headers.append(
(b"Sec-WebSocket-Protocol", ", ".join(subprotocols)))
if headers:
for header_name, header_value in headers:
conn.putheader(header_name, header_value)
conn.endheaders()
# Read out the response
try:
response = conn.getresponse()
except socket.timeout:
# See if they left an exception for us to load
test_app.get_received()
raise RuntimeError(
"Daphne timed out handling request, no exception found.")
# Check we got a good response code
if response.status != 101:
raise RuntimeError(
"WebSocket upgrade did not result in status code 101")
# Prepare headers for subprotocol searching
response_headers = dict(
(n.lower(), v) for n, v in response.getheaders())
response.read()
assert not response.closed
# Return the raw socket and any subprotocol
return conn.sock, response_headers.get("sec-websocket-protocol", None)
class http_object(url_object):
def __init__(self,machine,urlpath):
self.machine = machine
self.urlpath = urlpath
self.http = HTTPConnection(self.machine)
try:
self.http = HTTPConnection(self.machine)
except:
error('Error! accessing server', self.machine)
self.http.putrequest('GET',self.urlpath)
self.http.putheader('Accept','*/*')
self.http.endheaders()
self.response = self.http.getresponse()
#errcode, errmesg, self.headers = self.http.getreply()
if self.response.status != 200:
self.http.close()
error('Error! Accessing url on the server.',res.status,res.reason)
filesize = int(self.response.getheader('content-length'))
if filesize < 2000 :
self.http.close()
error('Error! Accessing url on the server. bytes-received :',filesize)
def __del__(self):
self.http.close()
def gettime(self):
# Get the remote timestamps
urltimestamp = self.response.getheader('last-modified')
month_d = {'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6,
'Jul':7,'Aug':8,'Sep':9,'Oct':10,'Nov':11,'Dec':12}
urltimesplit = urltimestamp.split()
if len(urltimesplit) == 6 : #Sun, 06 Nov 1994 08:49:37 GMT
day,month,year,time = urltimesplit[1:-1]
elif len(urltimesplit) == 4 : #Sunday, 06-Nov-94 08:49:37 GMT
time = urltimesplit[2]
day,month,year = urltimesplits[1].split('-')
else : #Sun Nov 6 08:49:37 1994
month,day,time,year = urlsplit[1:]
hour,min,sec = time.split(':')
newtime = (int(year),month_d[month],int(day),int(hour),
int(min),int(sec),-1,-1,0)
self.remotetime = mktime(newtime) - timezone
return self.remotetime
def getfile(self,outfilename):
#read the data
if False:
f = self.http.getfile()
data = f.read()
f.close()
data = self.response.read()
# Now write this data to a file
fp = open(outfilename,'wb')
fp.write(data)
fp.close()
def post(cookie):
web = HTTPConnection('localhost', 8118)
web.set_tunnel('158.69.76.135')
web.connect()
web.putrequest('POST', '/level4.php')
web.putheader('Cookie', 'HoldTheDoor=' + cookie)
web.putheader('Content-Type', 'application/x-www-form-urlencoded')
web.putheader('Referer', 'http://158.69.76.135/level4.php')
body = b'id=701&holdthedoor=Submit&key=' + cookie.encode('ASCII')
web.putheader('Content-Length', str(len(body)))
web.endheaders()
web.send(body)
response = web.getresponse()
web.close()
def run_daphne_http(
self,
method,
path,
params,
body,
responses,
headers=None,
timeout=1,
xff=False,
request_buffer_size=None,
):
"""
Runs Daphne with the given request callback (given the base URL)
and response messages.
"""
with DaphneTestingInstance(
xff=xff, request_buffer_size=request_buffer_size) as test_app:
# Add the response messages
test_app.add_send_messages(responses)
# Send it the request. We have to do this the long way to allow
# duplicate headers.
conn = HTTPConnection(test_app.host,
test_app.port,
timeout=timeout)
if params:
path += "?" + parse.urlencode(params, doseq=True)
conn.putrequest(method,
path,
skip_accept_encoding=True,
skip_host=True)
# Manually send over headers
if headers:
for header_name, header_value in headers:
conn.putheader(header_name, header_value)
# Send body if provided.
if body:
conn.putheader("Content-Length", str(len(body)))
conn.endheaders(message_body=body)
else:
conn.endheaders()
try:
response = conn.getresponse()
except socket.timeout:
# See if they left an exception for us to load
test_app.get_received()
raise RuntimeError(
"Daphne timed out handling request, no exception found.")
# Return scope, messages, response
return test_app.get_received() + (response, )
def query_worlds(url, gamename):
u = urlparse(url)
xml = """<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<soap:Body>
<GetDatacenters xmlns="http://www.turbine.com/SE/GLS">
<game>%s</game>
</GetDatacenters>
</soap:Body>
</soap:Envelope>
""" % (gamename)
c = HTTPConnection(u.netloc, 80)
c.putrequest("POST", u.path)
c.putheader("Content-Type", "text/xml; charset=utf-8")
c.putheader("SOAPAction", "http://www.turbine.com/SE/GLS/GetDatacenters")
c.putheader("Content-Length", str(len(xml)))
c.endheaders()
c.send(bytes(xml, "utf-8"))
r = c.getresponse()
if r.getcode() is not 200:
raise RuntimeError("HTTP post failed.")
rdata = r.read().decode("utf-8")
rdata = strip_namespaces(rdata)
xml = ElementTree.fromstring(rdata)
datacenters = xml.findall("Body/GetDatacentersResponse/GetDatacentersResult/*")
for dc in datacenters:
authserver = dc.find('AuthServer').text
patchserver = dc.find('PatchServer').text
config = dc.find('LauncherConfigurationServer').text
worlds = dc.findall("Worlds/*")
w = []
for world in worlds:
neu = {"name": world.find("Name").text,
"login": world.find("LoginServerUrl").text,
"chat": world.find("ChatServerUrl").text,
"language": world.find("Language").text,
"status": world.find("StatusServerUrl").text}
neu = query_host(neu)
w.append(neu)
return (w, authserver, patchserver, config)
raise RuntimeError("Failed to parse response from login server.")
def use_http_client():
from http.client import HTTPConnection
conn = HTTPConnection('ip.cn', 80)
conn.set_debuglevel(1)
conn.connect()
conn.putrequest('GET', '/')
conn.putheader(
'User-Agent',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
)
conn.endheaders()
res = conn.getresponse()
print('\n', res.read().decode('utf-8'))
conn.close()
def _checkURL(self, url):
"""
Check if the ``url`` is *alive* (i.e., remote server returns code 200(OK)
or 401 (unauthorized)).
"""
try:
p = urlparse(url)
h = HTTPConnection(p[1])
h.putrequest('HEAD', p[2])
h.endheaders()
r = h.getresponse()
# CloudMan UI is pwd protected so include 401
return r.status in (200, 401)
except Exception:
# No response or no good response
return False
def wait(self, timeout=TIMEOUT):
self._logger.log("TestServer %s:%s shutting down" % self._server_addr)
self.request_count = 0
if self._thread.is_alive():
client = HTTPConnection("127.0.0.1:%s" % self._client_port,
timeout=TIMEOUT)
client.putrequest("POST", "/SHUTDOWN")
client.putheader("Content-Length", "0")
client.endheaders()
# 13 == len('Server Closed')
client.getresponse().read(13)
client.close()
self._thread.join(timeout=TIMEOUT)
if self._server:
self._server.server_close()
self.request_count = self._server.request_count
del self._server
sleep(0.5) # fudge factor allow socket close to complete
def solr_commit():
"""
commit changes
"""
DATA = '<commit/>'
DATA = DATA.encode("utf-8")
con = HTTPConnection('0.0.0.0:8983')
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(DATA)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.endheaders()
con.send(DATA)
r = con.getresponse()
if str(r.status) == '200':
print(r.read())
else:
print(r.status)
print(r.read())
def solr_add(type, id):
"""
Add a document to index
"""
DATA = atpic.solr_sqlbased.solr_generate(type, id)
DATA = DATA.encode("utf-8")
con = HTTPConnection('0.0.0.0:8983')
con.putrequest('POST', '/solr/update/')
con.putheader('content-length', str(len(DATA)))
con.putheader('content-type', 'text/xml; charset=UTF-8')
con.putheader('connection', 'close')
con.endheaders()
con.send(DATA)
r = con.getresponse()
if str(r.status) == '200':
print(r.read())
else:
print(r.status)
print(r.read())
def putrequest(self, method, url, *args, **kwargs):
""""""
# Empty docstring because the indentation of CPython's implementation
# is broken but we don't want this method in our documentation.
match = _CONTAINS_CONTROL_CHAR_RE.search(method)
if match:
raise ValueError(
f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
)
return _HTTPConnection.putrequest(self, method, url, *args, **kwargs)
def test_http_over_https(self):
if self.scheme != 'https':
return self.skip('skipped (not running HTTPS)... ')
# Try connecting without SSL.
conn = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
conn.putrequest('GET', '/', skip_host=True)
conn.putheader('Host', self.HOST)
conn.endheaders()
response = conn.response_class(conn.sock, method='GET')
try:
response.begin()
self.assertEqual(response.status, 400)
self.body = response.read()
self.assertBody('The client sent a plain HTTP request, but this '
'server only speaks HTTPS on this port.')
except socket.error:
e = sys.exc_info()[1]
# "Connection reset by peer" is also acceptable.
if e.errno != errno.ECONNRESET:
raise
def test_http_over_https(self):
if self.scheme != 'https':
return self.skip('skipped (not running HTTPS)... ')
# Try connecting without SSL.
conn = HTTPConnection('%s:%s' % (self.interface(), self.PORT))
conn.putrequest('GET', '/', skip_host=True)
conn.putheader('Host', self.HOST)
conn.endheaders()
response = conn.response_class(conn.sock, method='GET')
try:
response.begin()
self.assertEqual(response.status, 400)
self.body = response.read()
self.assertBody('The client sent a plain HTTP request, but this '
'server only speaks HTTPS on this port.')
except socket.error:
e = sys.exc_info()[1]
# "Connection reset by peer" is also acceptable.
if e.errno != errno.ECONNRESET:
raise
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(self,
host='api.dynect.net',
port=443,
ssl=True,
api_version="current",
proxy_host=None,
proxy_port=None):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
# Add support for proxy connections
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.use_proxy = False
self.proxied_root = None
if proxy_host is not None and proxy_port is not None:
self.use_proxy = True
scheme = 'https' if self.ssl else 'http'
self.proxied_root = "{}://{}".format(scheme, self.host)
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def _connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def _proxy_connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.proxy_host, self.proxy_port)
else:
msg = "Establishing SSL connection to %s:%s\n" % (self.proxy_host,
self.proxy_port)
self._debug(msg)
self._conn = HTTPConnection(self.proxy_host, self.proxy_port)
self._conn.set_tunnel(self.host, self.port)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.use_proxy:
self._proxy_connect()
else:
self._connect()
def execute(self, uri, method, args=None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods))
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in list(headers.items()):
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
class HTTP:
def __init__(self, host, timeout = CONFIG.TIME_OUT, proxy = CONFIG.PROXY, user_agents_type = ''):
self.timeout = timeout
self.proxy = proxy
self.store_cookie = False
self.redirect = True #flag redirect when check found Location in header
self.rand_useragent = True #random useragents flag
if user_agents_type == 'bot':
self.user_agents = Bot_User_Agent()
else:
self.user_agents = User_Agent()
self.user_agent = self.user_agents.user_agent
HTTPConnection.debuglevel = 10
######################################
self.url = ''
self.method = 'GET'
self.headers = {
'User-Agent': self.user_agent,
'Accept-Encoding': 'gzip, deflate',
'Connection': 'close'
}
######################################
https = False
if self.proxy != None:
host = proxy
if host.startswith('http'):
parser = parse.urlparse(host)
self.host = parser.hostname
self.port = parser.port
if self.port == None:
if parser.scheme == 'https':
self.port = 443
else:
self.port = 80
if parser.scheme == 'https':
https = True
_user = parser.username if parser.username else ''
_pass = parser.password if parser.password else ''
_login = str(_user) + ':' + str(_pass)
else:
spliter = host.split(':')
self.host = spliter[0]
if len(spliter) < 2:
self.port = 80
if len(_login) > 1:
self.headers.update({'Authorization': "Basic " + b64encode(_login.encode('ascii')).decode('utf-8')})
if https:
self.request = HTTPSConnection(self.host,self.port, timeout = self.timeout)
else:
self.request = HTTPConnection(self.host,self.port, timeout = self.timeout)
def init(self, url, method , header = {}):
try:
if method:
self.method = method
# proxy setting
if not self.proxy and url.startswith('http'):
url = url.split('/', 3)
if len(url) > 3:
self.url = '/' + url[3]
else:
self.url = url
#random user agent
if self.rand_useragent:
self.user_agent = self.user_agents.getRandomUserAgent()
self.headers.update({'User-Agent': self.user_agent})
#update header for POST method
if self.method == 'POST':
self.headers.update({'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'})
elif 'Content-Type' in self.headers:
del self.headers['Content-Type']
# update header
if len(header) > 0:
self.headers.update(header)
# put all paramater request
self.request.putrequest(self.method, self.url)
# put all request header
for hk, hv in self.headers.items():
self.request.putheader(hk, hv)
# add \r\n\r\n end of header
self.request.endheaders()
# set timeout request
self.request.sock.settimeout(self.timeout)
except Exception as e:
raise
def send(self, data =b''):
if data:
self.request.send(data)
def recv(self, chunk = 0):
### get response header
self.response = self.request.getresponse()
### store cookie
if self.store_cookie and self.response.getheader('set-cookie'):
cookies = dict()
cook = self.request.getheader['cookie'] + '; ' + self.response.getheader('set-cookie')
for c in cook.split(';'):
tmp = c.trim().split('=', 1)
if len(tmp) == 2:
cookies.update({tmp[0],tmp[1]})
self.request.putheader('cookie', '; '.join(k +'='+ v for k,v in cookies.items()))
### yield all data
if chunk > 0:
while not self.response.closed:
yield self.response.read(chunk)
else:
result = self.response.read()
print(self.response.getheaders())
### extract gzip file, this method not support for send chunk data
if self.response.getheader('content-encoding') == 'gzip':
result = GzipFile(fileobj=BytesIO(result)).read()
### auto redirect, this method not support for send chunk data
if self.redirect:
refresh = search(b'http-equiv=.refresh(.*?)content=("|\')(.*?)url=(.*?)\2', result[:1000].lower())
if(self.response.getheader('location') or refresh):
self.refresh = True
if self.response.getheader('location'):
self.url = self.response.getheader('location')
else:
self.url = refresh.group(4)
self.params = ''
self.method = 'GET'
if 'Content-Type' in self.headers:
del self.headers['Content-Type']
### yield all content
yield result
self.request.close()
def Request(self, url, method = 'GET', params = b'', header = {}):
self.refresh = False
self.params = params
if type(self.params) == type(""):
self.params = self.params.encode()
if method.upper() == "POST":
header.update({"Content-Length": len(self.params)})
while True:
self.init(url, method, header)
self.send(self.params)
self.result = b''
for data in self.recv():
self.result += data
if not self.refresh:
# print(self.result)
return self.result
else:
self.refresh = False
# def Request(self, url, method = 'GET', params = None, header = {}, chunk = 0):
# # print("data : %s \nparams : %s" % (url,params))
# if self.rand_useragent:
# self.user_agent = self.user_agents.getRandomUserAgent()
# self.headers.update({'User-Agent': self.user_agent})
# if params:
# self.headers.update({'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'})
# elif 'Content-Type' in self.headers:
# del self.headers['Content-Type']
# if len(header) > 0:
# self.headers.update(header)
# while True:
# if not self.proxy:
# if url.startswith('http'):
# url = url.split('/', 3)
# if len(url) > 3:
# url = '/' + url[3]
# if params:
# self.request.request(method, url, params, headers = self.headers)
# else:
# self.request.request(method, url, headers = self.headers)
# self.request.sock.settimeout(self.timeout)
# self.response = self.request.getresponse()
# ######## store cookie ###########
# if self.storecookie and self.response.getheader('set-cookie'):
# cookies = []
# for c in self.response.getheader('set-cookie').split(','):
# cookies.append(c.split(';', 1)[0])
# cookie = self.headers['Cookie'] + ';' if 'Cookie' in self.headers else ''
# cookies = cookie + '; '.join(cookies)
# result = {}
# for c in cookies.split(';'):
# tmp = c.strip().split('=', 1)
# if len(tmp) == 2:
# result[tmp[0]] = tmp[1]
# self.cookie = '; '.join(k +'='+ v for k,v in result.items())
# self.headers.update({'Cookie': self.cookie})
# ###
# if chunk > 0:
# break
# self.result = self.response.read()
# if self.response.getheader('content-encoding') == 'gzip':
# self.result = GzipFile(fileobj=BytesIO(self.result)).read()
# # self.result = self.result.decode('utf-8', 'replace')
# ###
# refresh = search(b'http-equiv=.refresh(.*?)content=("|\')(.*?)url=(.*?)\2',self.result[:1000].lower())
# if (self.response.getheader('location') or refresh) and self.redirect:
# if self.response.getheader('location'):
# url = self.response.getheader('location')
# else:
# url = refresh.group(4)
# params = None
# method = 'GET'
# if 'Content-Type' in self.headers:
# del self.headers['Content-Type']
# else:
# break
# #########################
# if chunk > 0:
# while not self.response.closed:
# yield self.response.read(chunk)
# self.request.close()
# else:
# self.request.close()
# yield self.result
class http_object(url_object):
def __init__(self, machine, urlpath):
self.machine = machine
self.urlpath = urlpath
self.http = HTTPConnection(self.machine)
try:
self.http = HTTPConnection(self.machine)
except:
error('Error! accessing server', self.machine)
self.http.putrequest('GET', self.urlpath)
self.http.putheader('Accept', '*/*')
self.http.endheaders()
self.response = self.http.getresponse()
#errcode, errmesg, self.headers = self.http.getreply()
if self.response.status != 200:
self.http.close()
error('Error! Accessing url on the server.', res.status,
res.reason)
filesize = int(self.response.getheader('content-length'))
if filesize < 2000:
self.http.close()
error('Error! Accessing url on the server. bytes-received :',
filesize)
def __del__(self):
self.http.close()
def gettime(self):
# Get the remote timestamps
urltimestamp = self.response.getheader('last-modified')
month_d = {
'Jan': 1,
'Feb': 2,
'Mar': 3,
'Apr': 4,
'May': 5,
'Jun': 6,
'Jul': 7,
'Aug': 8,
'Sep': 9,
'Oct': 10,
'Nov': 11,
'Dec': 12
}
urltimesplit = urltimestamp.split()
if len(urltimesplit) == 6: #Sun, 06 Nov 1994 08:49:37 GMT
day, month, year, time = urltimesplit[1:-1]
elif len(urltimesplit) == 4: #Sunday, 06-Nov-94 08:49:37 GMT
time = urltimesplit[2]
day, month, year = urltimesplits[1].split('-')
else: #Sun Nov 6 08:49:37 1994
month, day, time, year = urlsplit[1:]
hour, min, sec = time.split(':')
newtime = (int(year), month_d[month], int(day), int(hour), int(min),
int(sec), -1, -1, 0)
self.remotetime = mktime(newtime) - timezone
return self.remotetime
def getfile(self, outfilename):
#read the data
if False:
f = self.http.getfile()
data = f.read()
f.close()
data = self.response.read()
# Now write this data to a file
fp = open(outfilename, 'wb')
fp.write(data)
fp.close()
def upload(self, filename):
content = open(filename, 'rb').read()
meta = self.distribution.metadata
data = {
':action': 'doc_upload',
'name': meta.get_name(),
'content': (os.path.basename(filename), content),
}
# set up the authentication
auth = "Basic " + base64.encodestring(self.username + ":" +
self.password).strip()
# Build up the MIME payload for the POST data
boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
sep_boundary = '\n--' + boundary
end_boundary = sep_boundary + '--'
body = StringIO.StringIO()
for key, value in list(data.items()):
# handle multiple entries for the same name
if type(value) != type([]):
value = [value]
for value in value:
if type(value) is tuple:
fn = ';filename="%s"' % value[0]
value = value[1]
else:
fn = ""
value = str(value)
body.write(sep_boundary)
body.write('\nContent-Disposition: form-data; name="%s"' % key)
body.write(fn)
body.write("\n\n")
body.write(value)
if value and value[-1] == '\r':
body.write('\n') # write an extra newline (lurve Macs)
body.write(end_boundary)
body.write("\n")
body = body.getvalue()
self.announce("Submitting documentation to %s" % (self.repository),
log.INFO)
# build the Request
# We can't use urllib2 since we need to send the Basic
# auth right with the first request
schema, netloc, url, params, query, fragments = \
urllib.parse.urlparse(self.repository)
assert not params and not query and not fragments
if schema == 'http':
http = HTTPConnection(netloc)
elif schema == 'https':
http = HTTPSConnection(netloc)
else:
raise AssertionError("unsupported schema " + schema)
data = ''
loglevel = log.INFO
try:
http.connect()
http.putrequest("POST", url)
http.putheader('Content-type',
'multipart/form-data; boundary=%s' % boundary)
http.putheader('Content-length', str(len(body)))
http.putheader('Authorization', auth)
http.endheaders()
http.send(body)
except socket.error as e:
self.announce(str(e), log.ERROR)
return
response = http.getresponse()
if response.status == 200:
self.announce(
'Server response (%s): %s' %
(response.status, response.reason), log.INFO)
elif response.status == 301:
location = response.getheader('Location')
if location is None:
location = 'http://packages.python.org/%s/' % meta.get_name()
self.announce('Upload successful. Visit %s' % location, log.INFO)
else:
self.announce(
'Upload failed (%s): %s' % (response.status, response.reason),
log.ERROR)
if self.show_response:
print(('-' * 75, response.read(), '-' * 75))
