def query_host(world): u = urlparse(world['status']) c = HTTPConnection(u.netloc, 80) c.putrequest("GET", u.path + '?' + u.query) c.putheader("Content-Type", "text/xml; charset=utf-8") c.endheaders() r = c.getresponse() if r.getcode() is not 200: raise RuntimeError("Failed to query information about the server.") rdata = r.read().decode("utf-8") if len(rdata) is 0: print("The given server appears to be down.") exit(0) xml = ElementTree.fromstring(rdata) loginserver = xml.find("loginservers").text loginservers = loginserver.split(';') worldqueue = xml.find("queueurls").text worldqueues = worldqueue.split(';') world['host'] = world['login'] = loginservers[0] world['queue'] = worldqueues[0] return world
def _sendChunked(self, conn:client.HTTPConnection): conn.putrequest(self.method, self.selector, skip_accept_encoding=True) for header, value in self._headers.items(): conn.putheader(header, value) conn.endheaders() def send_line(line): chunk = line.rstrip() if chunk: if hasattr(chunk, 'encode'): # http.client API and HTTP spec conform encoding! chunk = chunk.encode('iso-8859-1') chunk = (b'%x\r\n' % len(chunk)) + chunk + b'\r\n' conn.send(chunk) if hasattr(self.body, 'readline'): line = self.body.readline() while line: send_line(line) line = self.body.readline() else: for line in self.body.splitlines(): send_line(line)
def send_encoded(self, url, data): url2 = urlparse(url) host = url2.netloc path = url2.path or '/' query = '?' + url2.query if url2.query else '' request = path + query data2 = '&'.join( (quote(k) + '=' + quote_from_bytes(str(v).encode('utf8')) for k, v in data.items())).encode('ascii') try: http = HTTPConnection(host) http.putrequest('POST', request) http.putheader('Content-Type', 'application/x-www-form-urlencoded') http.putheader('User-Agent', 'Fluxid MOC Scrobbler 0.2') http.putheader('Content-Length', str(len(data2))) http.endheaders() http.send(data2) response = http.getresponse().read().decode('utf8').upper().strip() except Exception as e: raise HardErrorException(str(e)) if response == 'BADSESSION': raise BadSessionException elif response.startswith('FAILED'): raise FailedException( response.split(' ', 1)[1].strip() + (' POST = [%r]' % data2))
def urlopen(url, svprev, formdata): ua = "SPlayer Build %d" % svprev #prepare data #generate a random boundary boundary = "----------------------------" + "%x" % random.getrandbits(48) data = [] for item in formdata: data.append("--" + boundary + "\r\nContent-Disposition: form-data; name=\"" + item[0] + "\"\r\n\r\n" + item[1] + "\r\n") data.append("--" + boundary + "--\r\n") data = "".join(data) cl = str(len(data)) r = urlparse(url) h = HTTPConnection(r.hostname) h.connect() h.putrequest("POST", r.path, skip_host=True, skip_accept_encoding=True) h.putheader("User-Agent", ua) h.putheader("Host", r.hostname) h.putheader("Accept", "*/*") h.putheader("Content-Length", cl) h.putheader("Expect", "100-continue") h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary) h.endheaders() h.send(data) resp = h.getresponse() if resp.status != OK: raise Exception("HTTP response " + str(resp.status) + ": " + resp.reason) return resp
def query_datacenters(game = "DDO", datacenterurl = "http://gls.ddo.com/GLS.DataCenterServer/Service.asmx"): url = urlparse(datacenterurl) soaprequest = """<?xml version="1.0" encoding="utf-8"?> <soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema"> <soap:Body> <GetDatacenters xmlns="http://www.turbine.com/SE/GLS"> <game>%s</game> </GetDatacenters> </soap:Body> </soap:Envelope> """ % (game) c = HTTPConnection(url.netloc, 80) c.putrequest("POST", url.path) c.putheader("Content-Type", "text/xml; charset=utf-8") c.putheader("SOAPAction", "http://www.turbine.com/SE/GLS/GetDatacenters") c.putheader("Content-Length", str(len(soaprequest))) c.endheaders() c.send(bytes(soaprequest, "utf-8")) r = c.getresponse() if r.getcode() is not 200: raise LoginError('Failed to query data center for information.') xml = _getxmlresponse(r) dcs = [] datacenters = xml.findall('Body/GetDatacentersResponse/GetDatacentersResult/*') for dc in datacenters: datacenter = DataCenter() datacenter._parse_xml(dc) dcs.append(datacenter) return dcs
def _query_details(self): try: u = urlparse(self._statusurl) c = HTTPConnection(u.netloc, u.port) c.putrequest("GET", u.path + '?' + u.query) c.putheader("Content-Type", "text/xml; charset=utf-8") c.endheaders() r = c.getresponse() if r.getcode() is not 200: raise LoginError("Failed to query information about the server.") xml = _getxmlresponse(r) loginserver = xml.find("loginservers").text self._loginservers = loginserver.split(';') if len(self._loginservers) == 0: raise LoginError('World provided no login server or servers.') worldqueue = xml.find("queueurls").text self._worldqueues = worldqueue.split(';') if len(self._worldqueues) == 0: raise LoginError('World provided no queue or queues.') self._down = False except xml.etree.ElementTree.ParseError as pe: # XML error means wrong or no reply: Server is down. self._down = True except: # Rethrow other exceptions raise
def send_encoded(self, url, data): url2 = urlparse(url) host = url2.netloc path = url2.path or '/' query = '?' + url2.query if url2.query else '' request = path + query data2 = '&'.join(( quote(k) + '=' + quote_from_bytes(str(v).encode('utf8')) for k, v in data.items() )).encode('ascii') try: http = HTTPConnection(host) http.putrequest('POST', request) http.putheader('Content-Type', 'application/x-www-form-urlencoded') http.putheader('User-Agent', 'Fluxid MOC Scrobbler 0.2') http.putheader('Content-Length', str(len(data2))) http.endheaders() http.send(data2) response = http.getresponse().read().decode('utf8').upper().strip() except Exception as e: raise HardErrorException(str(e)) if response == 'BADSESSION': raise BadSessionException elif response.startswith('FAILED'): raise FailedException(response.split(' ', 1)[1].strip() + (' POST = [%r]' % data2))
def websocket_handshake( self, test_app, path="/", params=None, headers=None, subprotocols=None, timeout=1, ): """ Runs a WebSocket handshake negotiation and returns the raw socket object & the selected subprotocol. You'll need to inject an accept or reject message before this to let it complete. """ # Send it the request. We have to do this the long way to allow # duplicate headers. conn = HTTPConnection(test_app.host, test_app.port, timeout=timeout) if params: path += "?" + parse.urlencode(params, doseq=True) conn.putrequest("GET", path, skip_accept_encoding=True, skip_host=True) # Do WebSocket handshake headers + any other headers if headers is None: headers = [] headers.extend([ (b"Host", b"example.com"), (b"Upgrade", b"websocket"), (b"Connection", b"Upgrade"), (b"Sec-WebSocket-Key", b"x3JJHMbDL1EzLkh9GBhXDw=="), (b"Sec-WebSocket-Version", b"13"), (b"Origin", b"http://example.com"), ]) if subprotocols: headers.append( (b"Sec-WebSocket-Protocol", ", ".join(subprotocols))) if headers: for header_name, header_value in headers: conn.putheader(header_name, header_value) conn.endheaders() # Read out the response try: response = conn.getresponse() except socket.timeout: # See if they left an exception for us to load test_app.get_received() raise RuntimeError( "Daphne timed out handling request, no exception found.") # Check we got a good response code if response.status != 101: raise RuntimeError( "WebSocket upgrade did not result in status code 101") # Prepare headers for subprotocol searching response_headers = dict( (n.lower(), v) for n, v in response.getheaders()) response.read() assert not response.closed # Return the raw socket and any subprotocol return conn.sock, response_headers.get("sec-websocket-protocol", None)
class http_object(url_object): def __init__(self,machine,urlpath): self.machine = machine self.urlpath = urlpath self.http = HTTPConnection(self.machine) try: self.http = HTTPConnection(self.machine) except: error('Error! accessing server', self.machine) self.http.putrequest('GET',self.urlpath) self.http.putheader('Accept','*/*') self.http.endheaders() self.response = self.http.getresponse() #errcode, errmesg, self.headers = self.http.getreply() if self.response.status != 200: self.http.close() error('Error! Accessing url on the server.',res.status,res.reason) filesize = int(self.response.getheader('content-length')) if filesize < 2000 : self.http.close() error('Error! Accessing url on the server. bytes-received :',filesize) def __del__(self): self.http.close() def gettime(self): # Get the remote timestamps urltimestamp = self.response.getheader('last-modified') month_d = {'Jan':1,'Feb':2,'Mar':3,'Apr':4,'May':5,'Jun':6, 'Jul':7,'Aug':8,'Sep':9,'Oct':10,'Nov':11,'Dec':12} urltimesplit = urltimestamp.split() if len(urltimesplit) == 6 : #Sun, 06 Nov 1994 08:49:37 GMT day,month,year,time = urltimesplit[1:-1] elif len(urltimesplit) == 4 : #Sunday, 06-Nov-94 08:49:37 GMT time = urltimesplit[2] day,month,year = urltimesplits[1].split('-') else : #Sun Nov 6 08:49:37 1994 month,day,time,year = urlsplit[1:] hour,min,sec = time.split(':') newtime = (int(year),month_d[month],int(day),int(hour), int(min),int(sec),-1,-1,0) self.remotetime = mktime(newtime) - timezone return self.remotetime def getfile(self,outfilename): #read the data if False: f = self.http.getfile() data = f.read() f.close() data = self.response.read() # Now write this data to a file fp = open(outfilename,'wb') fp.write(data) fp.close()
def post(cookie): web = HTTPConnection('localhost', 8118) web.set_tunnel('158.69.76.135') web.connect() web.putrequest('POST', '/level4.php') web.putheader('Cookie', 'HoldTheDoor=' + cookie) web.putheader('Content-Type', 'application/x-www-form-urlencoded') web.putheader('Referer', 'http://158.69.76.135/level4.php') body = b'id=701&holdthedoor=Submit&key=' + cookie.encode('ASCII') web.putheader('Content-Length', str(len(body))) web.endheaders() web.send(body) response = web.getresponse() web.close()
def run_daphne_http( self, method, path, params, body, responses, headers=None, timeout=1, xff=False, request_buffer_size=None, ): """ Runs Daphne with the given request callback (given the base URL) and response messages. """ with DaphneTestingInstance( xff=xff, request_buffer_size=request_buffer_size) as test_app: # Add the response messages test_app.add_send_messages(responses) # Send it the request. We have to do this the long way to allow # duplicate headers. conn = HTTPConnection(test_app.host, test_app.port, timeout=timeout) if params: path += "?" + parse.urlencode(params, doseq=True) conn.putrequest(method, path, skip_accept_encoding=True, skip_host=True) # Manually send over headers if headers: for header_name, header_value in headers: conn.putheader(header_name, header_value) # Send body if provided. if body: conn.putheader("Content-Length", str(len(body))) conn.endheaders(message_body=body) else: conn.endheaders() try: response = conn.getresponse() except socket.timeout: # See if they left an exception for us to load test_app.get_received() raise RuntimeError( "Daphne timed out handling request, no exception found.") # Return scope, messages, response return test_app.get_received() + (response, )
def query_worlds(url, gamename): u = urlparse(url) xml = """<?xml version="1.0" encoding="utf-8"?> <soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema"> <soap:Body> <GetDatacenters xmlns="http://www.turbine.com/SE/GLS"> <game>%s</game> </GetDatacenters> </soap:Body> </soap:Envelope> """ % (gamename) c = HTTPConnection(u.netloc, 80) c.putrequest("POST", u.path) c.putheader("Content-Type", "text/xml; charset=utf-8") c.putheader("SOAPAction", "http://www.turbine.com/SE/GLS/GetDatacenters") c.putheader("Content-Length", str(len(xml))) c.endheaders() c.send(bytes(xml, "utf-8")) r = c.getresponse() if r.getcode() is not 200: raise RuntimeError("HTTP post failed.") rdata = r.read().decode("utf-8") rdata = strip_namespaces(rdata) xml = ElementTree.fromstring(rdata) datacenters = xml.findall("Body/GetDatacentersResponse/GetDatacentersResult/*") for dc in datacenters: authserver = dc.find('AuthServer').text patchserver = dc.find('PatchServer').text config = dc.find('LauncherConfigurationServer').text worlds = dc.findall("Worlds/*") w = [] for world in worlds: neu = {"name": world.find("Name").text, "login": world.find("LoginServerUrl").text, "chat": world.find("ChatServerUrl").text, "language": world.find("Language").text, "status": world.find("StatusServerUrl").text} neu = query_host(neu) w.append(neu) return (w, authserver, patchserver, config) raise RuntimeError("Failed to parse response from login server.")
def use_http_client(): from http.client import HTTPConnection conn = HTTPConnection('ip.cn', 80) conn.set_debuglevel(1) conn.connect() conn.putrequest('GET', '/') conn.putheader( 'User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36' ) conn.endheaders() res = conn.getresponse() print('\n', res.read().decode('utf-8')) conn.close()
def _checkURL(self, url): """ Check if the ``url`` is *alive* (i.e., remote server returns code 200(OK) or 401 (unauthorized)). """ try: p = urlparse(url) h = HTTPConnection(p[1]) h.putrequest('HEAD', p[2]) h.endheaders() r = h.getresponse() # CloudMan UI is pwd protected so include 401 return r.status in (200, 401) except Exception: # No response or no good response return False
def wait(self, timeout=TIMEOUT): self._logger.log("TestServer %s:%s shutting down" % self._server_addr) self.request_count = 0 if self._thread.is_alive(): client = HTTPConnection("127.0.0.1:%s" % self._client_port, timeout=TIMEOUT) client.putrequest("POST", "/SHUTDOWN") client.putheader("Content-Length", "0") client.endheaders() # 13 == len('Server Closed') client.getresponse().read(13) client.close() self._thread.join(timeout=TIMEOUT) if self._server: self._server.server_close() self.request_count = self._server.request_count del self._server sleep(0.5) # fudge factor allow socket close to complete
def solr_commit(): """ commit changes """ DATA = '<commit/>' DATA = DATA.encode("utf-8") con = HTTPConnection('0.0.0.0:8983') con.putrequest('POST', '/solr/update/') con.putheader('content-length', str(len(DATA))) con.putheader('content-type', 'text/xml; charset=UTF-8') con.endheaders() con.send(DATA) r = con.getresponse() if str(r.status) == '200': print(r.read()) else: print(r.status) print(r.read())
def solr_add(type, id): """ Add a document to index """ DATA = atpic.solr_sqlbased.solr_generate(type, id) DATA = DATA.encode("utf-8") con = HTTPConnection('0.0.0.0:8983') con.putrequest('POST', '/solr/update/') con.putheader('content-length', str(len(DATA))) con.putheader('content-type', 'text/xml; charset=UTF-8') con.putheader('connection', 'close') con.endheaders() con.send(DATA) r = con.getresponse() if str(r.status) == '200': print(r.read()) else: print(r.status) print(r.read())
def putrequest(self, method, url, *args, **kwargs): """""" # Empty docstring because the indentation of CPython's implementation # is broken but we don't want this method in our documentation. match = _CONTAINS_CONTROL_CHAR_RE.search(method) if match: raise ValueError( f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})" ) return _HTTPConnection.putrequest(self, method, url, *args, **kwargs)
def test_http_over_https(self): if self.scheme != 'https': return self.skip('skipped (not running HTTPS)... ') # Try connecting without SSL. conn = HTTPConnection('%s:%s' % (self.interface(), self.PORT)) conn.putrequest('GET', '/', skip_host=True) conn.putheader('Host', self.HOST) conn.endheaders() response = conn.response_class(conn.sock, method='GET') try: response.begin() self.assertEqual(response.status, 400) self.body = response.read() self.assertBody('The client sent a plain HTTP request, but this ' 'server only speaks HTTPS on this port.') except socket.error: e = sys.exc_info()[1] # "Connection reset by peer" is also acceptable. if e.errno != errno.ECONNRESET: raise
class DynectRest(object): """ A class for interacting with the Dynect Managed DNS REST API. @ivar host: The host to connect to (defaults to api.dynect.net) @type host: C{str} @ivar port: The port to connect to (defaults to 443) @type port: C{int} @ivar ssl: A boolean indicating whether or not to use SSL encryption (defaults to True) @type ssl: C{bool} @ivar poll_incomplete: A boolean indicating whether we should continue to poll for a result if a job comes back as incomplete (defaults to True) @type poll_incomplete: C{bool} @ivar api_version: The version of the API to request (defaults to "current") @type api_version: C{str} """ def __init__(self, host='api.dynect.net', port=443, ssl=True, api_version="current", proxy_host=None, proxy_port=None): """ Basic initializer method @param host: The host to connect to @type host: C{str} @param port: The port to connect to @type port: C{int} @param ssl: A boolean indicating whether or not to use SSL encryption @type ssl: C{bool} """ self.host = host self.port = port self.ssl = ssl # Continue polling for response if a job comes back as incomplete? self.poll_incomplete = True self.verbose = False self.api_version = api_version self.content_type = "application/json" self._token = None self._conn = None self._last_response = None self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT')) # Add support for proxy connections self.proxy_host = proxy_host self.proxy_port = proxy_port self.use_proxy = False self.proxied_root = None if proxy_host is not None and proxy_port is not None: self.use_proxy = True scheme = 'https' if self.ssl else 'http' self.proxied_root = "{}://{}".format(scheme, self.host) def _debug(self, msg): """ Debug output. """ if self.verbose: sys.stderr.write(msg) def _connect(self): if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % (self.host, self.port) self._debug(msg) self._conn = HTTPSConnection(self.host, self.port) else: msg = "Establishing unencrypted connection to %s:%s\n" % ( self.host, self.port) self._debug(msg) self._conn = HTTPConnection(self.host, self.port) def _proxy_connect(self): if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % (self.host, self.port) self._debug(msg) self._conn = HTTPSConnection(self.proxy_host, self.proxy_port) else: msg = "Establishing SSL connection to %s:%s\n" % (self.proxy_host, self.proxy_port) self._debug(msg) self._conn = HTTPConnection(self.proxy_host, self.proxy_port) self._conn.set_tunnel(self.host, self.port) def connect(self): """ Establishes a connection to the REST API server as defined by the host, port and ssl instance variables """ if self._token: self._debug("Forcing logout from old session.\n") orig_value = self.poll_incomplete self.poll_incomplete = False self.execute('/REST/Session', 'DELETE') self.poll_incomplete = orig_value self._token = None self._conn = None if self.use_proxy: self._proxy_connect() else: self._connect() def execute(self, uri, method, args=None): """ Execute a commands against the rest server @param uri: The uri of the resource to access. /REST/ will be prepended if it is not at the beginning of the uri. @type uri: C{str} @param method: One of 'DELETE', 'GET', 'POST', or 'PUT' @type method: C{str} @param args: Any arguments to be sent as a part of the request @type args: C{dict} """ if self._conn == None: self._debug("No established connection\n") self.connect() # Make sure the command is prefixed by '/REST/' if not uri.startswith('/'): uri = '/' + uri if not uri.startswith('/REST'): uri = '/REST' + uri # Make sure the method is valid if method.upper() not in self._valid_methods: msg = "%s is not a valid HTTP method. Please use one of %s" % ( method, ", ".join(self._valid_methods)) raise ValueError(msg) # Prepare arguments if args is None: args = {} args = self.format_arguments(args) self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args)) # Send the command and deal with results self.send_command(uri, method, args) # Deal with the results response = self._conn.getresponse() body = response.read() self._last_response = response if self.poll_incomplete: response, body = self.poll_response(response, body) self._last_response = response if sys.version_info[0] == 2: ret_val = json.loads(body) elif sys.version_info[0] == 3: ret_val = json.loads(body.decode('UTF-8')) self._meta_update(uri, method, ret_val) return ret_val def _meta_update(self, uri, method, results): """ Private method, not intended for use outside the class """ # If we had a successful log in, update the token if uri.startswith('/REST/Session') and method == 'POST': if results['status'] == 'success': self._token = results['data']['token'] # Otherwise, if it's a successful logout, blank the token if uri.startswith('/REST/Session') and method == 'DELETE': if results['status'] == 'success': self._token = None def poll_response(self, response, body): """ Looks at a response from a REST command, and while indicates that the job is incomplete, poll for response """ while response.status == 307: time.sleep(1) uri = response.getheader('Location') self._debug("Polling %s\n" % uri) self.send_command(uri, "GET", '') response = self._conn.getresponse() body = response.read() return response, body def send_command(self, uri, method, args): """ Responsible for packaging up the API request and sending it to the server over the established connection @param uri: The uri of the resource to interact with @type uri: C{str} @param method: The HTTP method to use @type method: C{str} @param args: Encoded arguments to send to the server @type args: C{str} """ if '%' not in uri: uri = pathname2url(uri) self._conn.putrequest(method, uri) # Build headers headers = { 'Content-Type': self.content_type, 'API-Version': self.api_version, } if self._token is not None: headers['Auth-Token'] = self._token for key, val in list(headers.items()): self._conn.putheader(key, val) # Now the arguments self._conn.putheader('Content-length', '%d' % len(args)) self._conn.endheaders() if sys.version_info[0] == 2: self._conn.send(args) elif sys.version_info[0] == 3: self._conn.send(bytes(args, 'UTF-8')) def format_arguments(self, args): """ Converts the argument dictionary to the format needed to transmit the REST request. @param args: Arguments to be passed to the REST call @type args: C{dict} @return: The encoded string to send in the REST request body @rtype: C{str} """ args = json.dumps(args) return args
class HTTP: def __init__(self, host, timeout = CONFIG.TIME_OUT, proxy = CONFIG.PROXY, user_agents_type = ''): self.timeout = timeout self.proxy = proxy self.store_cookie = False self.redirect = True #flag redirect when check found Location in header self.rand_useragent = True #random useragents flag if user_agents_type == 'bot': self.user_agents = Bot_User_Agent() else: self.user_agents = User_Agent() self.user_agent = self.user_agents.user_agent HTTPConnection.debuglevel = 10 ###################################### self.url = '' self.method = 'GET' self.headers = { 'User-Agent': self.user_agent, 'Accept-Encoding': 'gzip, deflate', 'Connection': 'close' } ###################################### https = False if self.proxy != None: host = proxy if host.startswith('http'): parser = parse.urlparse(host) self.host = parser.hostname self.port = parser.port if self.port == None: if parser.scheme == 'https': self.port = 443 else: self.port = 80 if parser.scheme == 'https': https = True _user = parser.username if parser.username else '' _pass = parser.password if parser.password else '' _login = str(_user) + ':' + str(_pass) else: spliter = host.split(':') self.host = spliter[0] if len(spliter) < 2: self.port = 80 if len(_login) > 1: self.headers.update({'Authorization': "Basic " + b64encode(_login.encode('ascii')).decode('utf-8')}) if https: self.request = HTTPSConnection(self.host,self.port, timeout = self.timeout) else: self.request = HTTPConnection(self.host,self.port, timeout = self.timeout) def init(self, url, method , header = {}): try: if method: self.method = method # proxy setting if not self.proxy and url.startswith('http'): url = url.split('/', 3) if len(url) > 3: self.url = '/' + url[3] else: self.url = url #random user agent if self.rand_useragent: self.user_agent = self.user_agents.getRandomUserAgent() self.headers.update({'User-Agent': self.user_agent}) #update header for POST method if self.method == 'POST': self.headers.update({'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'}) elif 'Content-Type' in self.headers: del self.headers['Content-Type'] # update header if len(header) > 0: self.headers.update(header) # put all paramater request self.request.putrequest(self.method, self.url) # put all request header for hk, hv in self.headers.items(): self.request.putheader(hk, hv) # add \r\n\r\n end of header self.request.endheaders() # set timeout request self.request.sock.settimeout(self.timeout) except Exception as e: raise def send(self, data =b''): if data: self.request.send(data) def recv(self, chunk = 0): ### get response header self.response = self.request.getresponse() ### store cookie if self.store_cookie and self.response.getheader('set-cookie'): cookies = dict() cook = self.request.getheader['cookie'] + '; ' + self.response.getheader('set-cookie') for c in cook.split(';'): tmp = c.trim().split('=', 1) if len(tmp) == 2: cookies.update({tmp[0],tmp[1]}) self.request.putheader('cookie', '; '.join(k +'='+ v for k,v in cookies.items())) ### yield all data if chunk > 0: while not self.response.closed: yield self.response.read(chunk) else: result = self.response.read() print(self.response.getheaders()) ### extract gzip file, this method not support for send chunk data if self.response.getheader('content-encoding') == 'gzip': result = GzipFile(fileobj=BytesIO(result)).read() ### auto redirect, this method not support for send chunk data if self.redirect: refresh = search(b'http-equiv=.refresh(.*?)content=("|\')(.*?)url=(.*?)\2', result[:1000].lower()) if(self.response.getheader('location') or refresh): self.refresh = True if self.response.getheader('location'): self.url = self.response.getheader('location') else: self.url = refresh.group(4) self.params = '' self.method = 'GET' if 'Content-Type' in self.headers: del self.headers['Content-Type'] ### yield all content yield result self.request.close() def Request(self, url, method = 'GET', params = b'', header = {}): self.refresh = False self.params = params if type(self.params) == type(""): self.params = self.params.encode() if method.upper() == "POST": header.update({"Content-Length": len(self.params)}) while True: self.init(url, method, header) self.send(self.params) self.result = b'' for data in self.recv(): self.result += data if not self.refresh: # print(self.result) return self.result else: self.refresh = False # def Request(self, url, method = 'GET', params = None, header = {}, chunk = 0): # # print("data : %s \nparams : %s" % (url,params)) # if self.rand_useragent: # self.user_agent = self.user_agents.getRandomUserAgent() # self.headers.update({'User-Agent': self.user_agent}) # if params: # self.headers.update({'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'}) # elif 'Content-Type' in self.headers: # del self.headers['Content-Type'] # if len(header) > 0: # self.headers.update(header) # while True: # if not self.proxy: # if url.startswith('http'): # url = url.split('/', 3) # if len(url) > 3: # url = '/' + url[3] # if params: # self.request.request(method, url, params, headers = self.headers) # else: # self.request.request(method, url, headers = self.headers) # self.request.sock.settimeout(self.timeout) # self.response = self.request.getresponse() # ######## store cookie ########### # if self.storecookie and self.response.getheader('set-cookie'): # cookies = [] # for c in self.response.getheader('set-cookie').split(','): # cookies.append(c.split(';', 1)[0]) # cookie = self.headers['Cookie'] + ';' if 'Cookie' in self.headers else '' # cookies = cookie + '; '.join(cookies) # result = {} # for c in cookies.split(';'): # tmp = c.strip().split('=', 1) # if len(tmp) == 2: # result[tmp[0]] = tmp[1] # self.cookie = '; '.join(k +'='+ v for k,v in result.items()) # self.headers.update({'Cookie': self.cookie}) # ### # if chunk > 0: # break # self.result = self.response.read() # if self.response.getheader('content-encoding') == 'gzip': # self.result = GzipFile(fileobj=BytesIO(self.result)).read() # # self.result = self.result.decode('utf-8', 'replace') # ### # refresh = search(b'http-equiv=.refresh(.*?)content=("|\')(.*?)url=(.*?)\2',self.result[:1000].lower()) # if (self.response.getheader('location') or refresh) and self.redirect: # if self.response.getheader('location'): # url = self.response.getheader('location') # else: # url = refresh.group(4) # params = None # method = 'GET' # if 'Content-Type' in self.headers: # del self.headers['Content-Type'] # else: # break # ######################### # if chunk > 0: # while not self.response.closed: # yield self.response.read(chunk) # self.request.close() # else: # self.request.close() # yield self.result
class http_object(url_object): def __init__(self, machine, urlpath): self.machine = machine self.urlpath = urlpath self.http = HTTPConnection(self.machine) try: self.http = HTTPConnection(self.machine) except: error('Error! accessing server', self.machine) self.http.putrequest('GET', self.urlpath) self.http.putheader('Accept', '*/*') self.http.endheaders() self.response = self.http.getresponse() #errcode, errmesg, self.headers = self.http.getreply() if self.response.status != 200: self.http.close() error('Error! Accessing url on the server.', res.status, res.reason) filesize = int(self.response.getheader('content-length')) if filesize < 2000: self.http.close() error('Error! Accessing url on the server. bytes-received :', filesize) def __del__(self): self.http.close() def gettime(self): # Get the remote timestamps urltimestamp = self.response.getheader('last-modified') month_d = { 'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12 } urltimesplit = urltimestamp.split() if len(urltimesplit) == 6: #Sun, 06 Nov 1994 08:49:37 GMT day, month, year, time = urltimesplit[1:-1] elif len(urltimesplit) == 4: #Sunday, 06-Nov-94 08:49:37 GMT time = urltimesplit[2] day, month, year = urltimesplits[1].split('-') else: #Sun Nov 6 08:49:37 1994 month, day, time, year = urlsplit[1:] hour, min, sec = time.split(':') newtime = (int(year), month_d[month], int(day), int(hour), int(min), int(sec), -1, -1, 0) self.remotetime = mktime(newtime) - timezone return self.remotetime def getfile(self, outfilename): #read the data if False: f = self.http.getfile() data = f.read() f.close() data = self.response.read() # Now write this data to a file fp = open(outfilename, 'wb') fp.write(data) fp.close()
def upload(self, filename): content = open(filename, 'rb').read() meta = self.distribution.metadata data = { ':action': 'doc_upload', 'name': meta.get_name(), 'content': (os.path.basename(filename), content), } # set up the authentication auth = "Basic " + base64.encodestring(self.username + ":" + self.password).strip() # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' sep_boundary = '\n--' + boundary end_boundary = sep_boundary + '--' body = StringIO.StringIO() for key, value in list(data.items()): # handle multiple entries for the same name if type(value) != type([]): value = [value] for value in value: if type(value) is tuple: fn = ';filename="%s"' % value[0] value = value[1] else: fn = "" value = str(value) body.write(sep_boundary) body.write('\nContent-Disposition: form-data; name="%s"' % key) body.write(fn) body.write("\n\n") body.write(value) if value and value[-1] == '\r': body.write('\n') # write an extra newline (lurve Macs) body.write(end_boundary) body.write("\n") body = body.getvalue() self.announce("Submitting documentation to %s" % (self.repository), log.INFO) # build the Request # We can't use urllib2 since we need to send the Basic # auth right with the first request schema, netloc, url, params, query, fragments = \ urllib.parse.urlparse(self.repository) assert not params and not query and not fragments if schema == 'http': http = HTTPConnection(netloc) elif schema == 'https': http = HTTPSConnection(netloc) else: raise AssertionError("unsupported schema " + schema) data = '' loglevel = log.INFO try: http.connect() http.putrequest("POST", url) http.putheader('Content-type', 'multipart/form-data; boundary=%s' % boundary) http.putheader('Content-length', str(len(body))) http.putheader('Authorization', auth) http.endheaders() http.send(body) except socket.error as e: self.announce(str(e), log.ERROR) return response = http.getresponse() if response.status == 200: self.announce( 'Server response (%s): %s' % (response.status, response.reason), log.INFO) elif response.status == 301: location = response.getheader('Location') if location is None: location = 'http://packages.python.org/%s/' % meta.get_name() self.announce('Upload successful. Visit %s' % location, log.INFO) else: self.announce( 'Upload failed (%s): %s' % (response.status, response.reason), log.ERROR) if self.show_response: print(('-' * 75, response.read(), '-' * 75))