def testMessage(self): conn = HTTPConnection('localhost:8080') conn.request("GET", "/message?id=1") request = conn.getresponse() result = request.read() self.assertEqual(request.status, 200) self.assertGreater(result.find('<div class="messagebox">'), 0) conn.request("GET", "/message?id=2") request = conn.getresponse() result = request.read() self.assertEqual(request.status, 200) self.assertGreater(result.find('Please select an email message from the list'), 0) conn.request("GET", "/message?id=-2") request = conn.getresponse() result = request.read() self.assertEqual(request.status, 200) self.assertGreater(result.find('Please select an email message from the list'), 0) conn = HTTPConnection('localhost:8080') conn.request("GET", "/message?id=1&onlymsg=true") request = conn.getresponse() result = request.read() result.find('The HTML Body') self.assertEqual(request.status, 200) self.assertEqual(result, 'The HTML Body') conn.close()
def send_email(request): try: recipients = request.GET["to"].split(",") url = request.GET["url"] proto, server, path, query, frag = urlsplit(url) if query: path += "?" + query conn = HTTPConnection(server) conn.request("GET", path) try: # Python 2.7+, use buffering of HTTP responses resp = conn.getresponse(buffering=True) except TypeError: # Python 2.6 and older resp = conn.getresponse() assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason) rawData = resp.read() conn.close() message = MIMEMultipart() message["Subject"] = "Graphite Image" message["To"] = ", ".join(recipients) message["From"] = "composer@%s" % gethostname() text = MIMEText("Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(), url)) image = MIMEImage(rawData) image.add_header("Content-Disposition", "attachment", filename="composer_" + strftime("%b%d_%I%M%p.png")) message.attach(text) message.attach(image) s = SMTP(settings.SMTP_SERVER) s.sendmail("composer@%s" % gethostname(), recipients, message.as_string()) s.quit() return HttpResponse("OK") except: return HttpResponse(format_exc())
def run(self): if self.file: print "RUN THE THREAD CLIENT" dataFile = open(self.file, "r") cdata = dataFile.read() conn = HTTPConnection(self.proxy) conn.request("GET", self.url) resp = conn.getresponse() rdata = resp.read() if rdata == cdata: self.result = True self.data = rdata conn.close() dataFile.close() print "CONNECTION CLOSE" else: conn = HTTPConnection(self.proxy) conn.request("GET", self.url) resp = conn.getresponse() rdata = resp.read() if resp.status == httplib.OK: self.result = True conn.close()
def test_get(self): PORT = TestMemHTTPServer.PORT server = MemHTTPServer(('localhost', PORT)) server.set_get_output('asdf', 'text/html', 'ASDF') server.set_get_output('qwer', 'text/plain', 'QWER') server.server_activate() client = HTTPConnection('localhost', PORT) client.connect() client.request('GET', 'asdf') server.handle_request() response = client.getresponse() self.assertEqual(200, response.status) self.assertEqual('ASDF', response.read()) self.assertEqual('text/html', response.getheader('Content-type')) client = HTTPConnection('localhost', PORT) client.connect() client.request('GET', 'qwer') server.handle_request() response = client.getresponse() self.assertEqual(200, response.status) self.assertEqual('QWER', response.read()) self.assertEqual('text/plain', response.getheader('Content-type'))
def _run_server(self, app): """Run a wsgi server in a separate thread""" ip, port = get_free_port() self.app = app = WSGIApplication(app, (ip, port)) def run(): logger = logging.getLogger("SeleniumWebDriverApp") def log_message(self, format, *args): logger.info("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), format % args)) # monkey patch to redirect request handler logs WSGIRequestHandler.log_message = log_message httpd = simple_server.make_server(ip, port, app, server_class=WSGIServer, handler_class=WSGIRequestHandler) httpd.serve_forever() app.thread = Process(target=run) app.thread.start() conn = HTTPConnection(ip, port) time.sleep(.5) for i in range(100): try: conn.request('GET', '/__application__') conn.getresponse() except (socket.error, CannotSendRequest): time.sleep(.3) else: break
def download_default_pages(names, prefix): from httplib import HTTPConnection host = 'trac.edgewall.org' if prefix and not prefix.endswith('/'): prefix += '/' conn = HTTPConnection(host) for name in names: if name in ('WikiStart', 'SandBox'): continue sys.stdout.write('Downloading %s%s' % (prefix, name)) conn.request('GET', '/wiki/%s%s?format=txt' % (prefix, name)) response = conn.getresponse() content = response.read() if prefix and (response.status != 200 or not content): sys.stdout.write(' %s' % name) conn.request('GET', '/wiki/%s?format=txt' % name) response = conn.getresponse() content = response.read() if response.status == 200 and content: with open('trac/wiki/default-pages/' + name, 'w') as f: lines = content.replace('\r\n', '\n').splitlines(True) f.write(''.join(line for line in lines if line.strip() != '[[TranslatedPages]]')) sys.stdout.write('\tdone.\n') else: sys.stdout.write('\tmissing or empty.\n') conn.close()
def fetchData ( server, markets, network ): marketsData = None networkData = None connection = HTTPConnection ( server, timeout=5 ) # markets connection.request ( 'GET', markets ) r = connection.getresponse ( ) if ( r.status == 200 ): marketsData = r.read ( ) else: stderr.write ( 'error: fetchData() got status %d for markets\n' % r.status ) return None # network connection.request ( 'GET', network ) r = connection.getresponse ( ) if ( r.status == 200 ): networkData = r.read ( ) else: stderr.write ( 'error: fetchData() got status %d for network\n' % r.status ) return None connection.close ( ) # parse JSON return ( json.loads(marketsData), json.loads(networkData), '' )
def run(self): conn = HTTPConnection(self.proxy) tmpFlag = True dataFile = open(self.file, "r") cdata = dataFile.read() dataFile = open(self.file2, "r") cdata2 = dataFile.read() conn.request("GET", self.url) resp = conn.getresponse() rdata = resp.read() if rdata != cdata: tmpFlag = False if resp.will_close == True: tmpFlag = False connHdrs = {"Connection": "close"} conn.request("GET", self.url2, headers=connHdrs) resp = conn.getresponse() rdata2 = resp.read() if rdata2 != cdata2: tmpFlag = False if resp.will_close == False: tmpFlag = False if tmpFlag == True: self.result = True conn.close() dataFile.close()
def run(self): if self.file: dataFile = open(self.file, "r") cdata = dataFile.read() conn = HTTPConnection(self.proxy) conn.request("GET", self.url) resp = conn.getresponse() rdata = resp.read() if rdata == cdata: self.result = True self.data = rdata conn.close() dataFile.close() else: conn = HTTPConnection(self.proxy) conn.request("GET", self.url) resp = conn.getresponse() rdata = resp.read() if resp.status == httplib.OK: self.result = True conn.close()
def run(self): ts = time() # print "Health check on", self.nodename conn = Conn(self.nodename, PORT, timeout=30) eltime = -1 error = "" try: conn.request("GET", "/esgf-nm/health-check-api?from=" + localhostname + "&forward=" + str(self.fwdcheck)) resp = conn.getresponse() if resp.status == 500: print "500 error" self.logger.error(resp.read()) eltime = time() - ts self.handle_resp(resp) except Exception as e: error = "connectivity problem" print e self.eltime = eltime nodemap_instance = get_instance() node_list = nodemap_instance.get_supernode_list() if not self.fwdcheck: if not self.checkarr is None: self.checkarr.append(self.nodename + "=" + str(eltime)) if (self.first): if len(node_list) > len(self.checkarr) + 2: sleep(.01) conn = Conn(self.fromnode, PORT, timeout=30) url = "/esgf-nm/health-check-rep?from=" + localhostname for n in self.checkarr: url = url + "&" + n error = "" resp = "" try: conn.request("GET", url) resp = conn.getresponse() self.handle_resp(resp) except Exception as e: error = "connectivity problem" print e
class HTTPRelayClient: def __init__(self, target): # Target comes as protocol://target:port/path self.target = target _, host, path = target.split(':') host = host[2:] self.path = '/' + path.split('/')[1] self.session = HTTPConnection(host) self.lastresult = None def sendNegotiate(self,negotiateMessage): #Check if server wants auth self.session.request('GET', self.path) res = self.session.getresponse() res.read() if res.status != 401: logging.info('Status code returned: %d. Authentication does not seem required for URL' % res.status) try: if 'NTLM' not in res.getheader('WWW-Authenticate'): logging.error('NTLM Auth not offered by URL, offered protocols: %s' % res.getheader('WWW-Authenticate')) return False except KeyError: logging.error('No authentication requested by the server for url %s' % self.target) return False #Negotiate auth negotiate = base64.b64encode(negotiateMessage) headers = {'Authorization':'NTLM %s' % negotiate} self.session.request('GET', self.path ,headers=headers) res = self.session.getresponse() res.read() try: serverChallenge = base64.b64decode(res.getheader('WWW-Authenticate')[5:]) return serverChallenge except (IndexError, KeyError): logging.error('No NTLM challenge returned from server') def sendAuth(self,authenticateMessageBlob, serverChallenge=None): #Negotiate auth auth = base64.b64encode(authenticateMessageBlob) headers = {'Authorization':'NTLM %s' % auth} self.session.request('GET', self.path,headers=headers) res = self.session.getresponse() if res.status == 401: return False else: logging.info('HTTP server returned error code %d, treating as a succesful login' % res.status) #Cache this self.lastresult = res.read() return True #SMB Relay server needs this @staticmethod def get_encryption_key(): return None
class CommandLineFu(object): def __init__(self): self._h = HTTP(API_DOMAIN) def _operate(self, action, search_limit): index = 0 search_result = [] while True: if search_limit != None and index > search_limit: break self._h.request("GET", action) index = index + 25 res = loads(self._h.getresponse().read()) if res == []: break else: search_result += res return search_result def browse(self, sort_order=SORT_BY_DATES, index = 0): """ Browse Commandlinefu history, sorted by dates or by votes. By default, CLF has 25 results per page, so index can be used to see others 25 results. """ self._h.request("GET","/commands/browse/%s/json/%d"%(sort_order,index)) return self._h.getresponse().read() def matching(self, match, sort_order = SORT_BY_DATES, search_limit = None): """ Search from history commands that match the "match" string. By default, it returns all result, sort by dates with no search limit. """ match64 = encodestring(match)[:-1] action = "/commands/matching/%s/%s/%s/json/%d"%(match, match64, sort_order, index) return self._operate(action, search_limit) def using(self, match, sort_order = SORT_BY_DATES, search_limit = None): """ Returns commands that implements the search word "match'. By default, it returns all result, sort by dates with no search limit. """ action = "/commands/using/%s/%s/json/%d"%(match, sort_order, index) return self._operate(action, search_limit) def by(self, match, sort_order = SORT_BY_DATES, search_limit = None): """ Returns commands from a specified user. By default, it returns all result, sort by dates with no search limit. """ action = "/commands/by/%s/%s/json/%d"%(match, sort_order, index) return self._operate(action, search_limit)
class AmbariClient(object): """Quick Ambari wrapper """ def __init__(self, host, port, user, password): self.conn = HTTPConnection(host=host, port=port) self.get_headers = { 'Authorization' : 'Basic %s' % b64encode('%s:%s' % (user, password)), 'Content-Type': 'application/json', 'X-Requested-By': 'ambari' } self.write_headers = { 'Authorization' : 'Basic %s' % b64encode('%s:%s' % (user, password)), 'Content-Type': 'application/x-www-form-urlencoded', 'X-Requested-By': 'ambari' } def get(self, endpoint): """GET with JSON result """ self.conn.request('GET', endpoint, headers=self.get_headers) response = self.conn.getresponse() if response.status not in xrange(200, 207): logger.debug('GET %s' % endpoint) logger.debug(response.read()) raise RuntimeError('Error connecting to %s, status=%s' % (endpoint, response.status)) content = response.read() if content: return json.loads(content) else: return None def _write(self, method, endpoint, data): self.conn.request(method, endpoint, headers=self.write_headers, body=data) response = self.conn.getresponse() if response.status not in xrange(200, 207): logger.debug('POST %s' % endpoint) logger.debug(response.read()) raise RuntimeError('Error connecting to %s, status=%s' % (endpoint, response.status)) content = response.read() if content: return json.loads(content) else: return None def put(self, endpoint, data): """PUT request """ self._write('PUT', endpoint, data) def post(self, endpoint, data): """POST request """ self._write('POST', endpoint, data)
def append_print_file(print_id, file_path, file_contents, apibase, password): """ Upload a file via the API append.php form input provision thingie. """ s, host, path, p, q, f = urlparse(apibase) host, port = (':' in host) and host.split(':') or (host, 80) query = urlencode({'print': print_id, 'password': password, 'dirname': dirname(file_path), 'mimetype': (guess_type(file_path)[0] or '')}) req = HTTPConnection(host, port) req.request('GET', path + '/append.php?' + query) res = req.getresponse() html = ElementTree.parse(res) for form in html.findall('*/form'): form_action = form.attrib['action'] inputs = form.findall('.//input') file_inputs = [input for input in inputs if input.attrib['type'] == 'file'] fields = [(input.attrib['name'], input.attrib['value']) for input in inputs if input.attrib['type'] != 'file' and 'name' in input.attrib] files = [(input.attrib['name'], basename(file_path), file_contents) for input in inputs if input.attrib['type'] == 'file'] if len(files) == 1: base_url = [el.text for el in form.findall(".//*") if el.get('id', '') == 'base-url'][0] resource_url = urljoin(base_url, file_path) post_type, post_body = encode_multipart_formdata(fields, files) s, host, path, p, query, f = urlparse(urljoin(apibase, form_action)) host, port = (':' in host) and host.split(':') or (host, 80) req = HTTPConnection(host, port) req.request('POST', path+'?'+query, post_body, {'Content-Type': post_type, 'Content-Length': str(len(post_body))}) res = req.getresponse() # res.read().startswith("Sorry, encountered error #1 ") assert res.status in range(200, 308), 'POST of file to %s resulting in status %s instead of 2XX/3XX' % (host, res.status) return resource_url raise Exception('Did not find a form with a file input, why is that?')
def setUp(self): from httplib import HTTPResponse with Stub() as HTTPResponse: HTTPResponse.status >> 200 HTTPResponse.read() >> '{"id": "THE-PRECIOUS-GUID", "title": "example title", "subtitle": "example title - subtitle", "description": "a description", "user": "******", "tags": [], "created_at": 1329767353.0, "source": "http://www.example.com/example.csv", "link": "http://www.junar.com/datastreams/some-url"}' with Stub() as conn: from httplib import HTTPConnection conn = HTTPConnection("api.junar.com") conn.request("POST", "/datastreams/publish", self.params, self.headers) conn.getresponse() >> HTTPResponse
def stopZope(port, auth): params = urlencode({'manage_shutdown:action': 'Shutdown'}) headers = { 'Content-Type': 'Application/x-www-form-urlencoded', 'Accept': 'text/plain', 'Authorization': 'Basic %s' % encodestring(auth) } conn = HTTPConnection('127.0.0.1:8080') conn.request('POST', '/Control_Panel', params, headers) # Throw the response away conn.getresponse().read() conn.close()
class ProductTest(TestCase): def setUp(self): self.conn = HTTPConnection('localhost', 5000, timeout=10) self.conn.set_debuglevel(DEBUG_LEVEL) def test_product_should_answer_200(self): self.conn.request(GET, '/product') response = self.conn.getresponse() self.assertEquals(response.status, 200) def test_product_id_should_answer_200(self): self.conn.request(GET, '/product/0') response = self.conn.getresponse() self.assertEquals(response.status, 200)
def _make_request(method, host, path, body='', headers={}): """ Makes an HTTP request. Args: method |str| = Type of request. GET, POST, PUT or DELETE. host |str| = The host and port of the server. E.g. vmpooler.myhost.com:8080 path |str| = The path of the url. E.g. /vm/vm_name body |str| = The body data to send with the request. headers |{str:str}| = Optional headers for the request. Returns: |HTTPResponse| = Response from the request. Raises: |RuntimeError| = If the vmpooler URL can't be reached """ try: conn = HTTPConnection(host) conn.request(method, path, body, headers) resp = conn.getresponse() return resp except gaierror: error = "Couldn't connect to address '{}'. Ensure this is the correct URL for " \ "the vmpooler".format(host) raise RuntimeError(error) except: print("Unkown error occured while trying to connect to {}".format(host)) raise
def _publish(args, server, uri): from httplib import HTTPConnection from urllib import urlencode args = dict((k,v) for k,v in args.items() if v != 'NA') uri = uri + "?" + urlencode(args) log.debug('Connect to: http://%s' % server) log.debug('GET %s' % uri) conn = HTTPConnection(server) if not conn: raise PublishException('Remote server connection error') conn.request("GET", uri) try: http = conn.getresponse() data = (http.status, http.reason, http.read()) conn.close() if not (data[0] == 200 and data[1] == 'OK'): raise PublishException('Server returned invalid status: %d %s %s' % data) except (DownloadError): logging.warning("Download error: ", sys.exc_info()[0]) data = None return data
def testGet(self): self.assertTrue(self.pasteThread.isAlive()) from httplib import HTTPConnection http_connection = HTTPConnection("localhost", port=self.port) http_connection.request("GET", "/api/Crawl") response = http_connection.getresponse() self.assertTrue(response.status == 404 or response.status == 200 or response.status == 500)
def request(self, method, endpoint, qargs={}, data=None): path = self.format_path(endpoint, qargs) conn = HTTPConnection('%s:80' % self.account) auth = b64encode("%s:%s" % (self.api_key, self.password)) headers = { 'Authorization': 'Basic %s' % auth, 'Content-Type': 'application/xml' } done = False while not done: try: conn.request(method, path, headers=headers, body=data) except socket.gaierror: if self.retry_on_socket_error: time.sleep(self.retry_timeout) continue else: raise resp = conn.getresponse() body = resp.read() if resp.status == 503 and self.retry_on_503: time.sleep(self.retry_timeout) else: done = True if 200 <= resp.status < 300: return body else: raise ApiError("%s request to %s returned: %s\n%s" % (method, path, resp.status, body), resp.status)
def search_on_playdeb(orig_file): global MIRROR_URL, GETDEB_SUBDIR http_connection = HTTPConnection(MIRROR_URL) basename = orig_file.split('_')[0] download_dir = 'http://' + MIRROR_URL + '/' + GETDEB_SUBDIR + '/ubuntu/pool/games/' + get_package_subdir(orig_file) + \ '/' + basename + '/' http_connection.request('GET', download_dir) http_response = http_connection.getresponse() if http_response.status != 200: return None data = http_response.read() http_connection.close() data = data.split('\n') package_lines = list() for line in data: if basename in line: package_lines.append(line) if len(package_lines) == 0: return None p_d = list() package_re = re.compile('<a .*?>(?P<orig>.*?)(?:\.diff\.gz|\.debian\.tar\.gz)<') download_re = re.compile('<a href="(?P<download>.*?)">') for line in package_lines: search_result = re.search(package_re, line) if not search_result: continue orig = search_result.group('orig') search_result = re.search(download_re, line) download = download_dir + search_result.group('download') p_d.append((orig,download)) return p_d
def send_email(request): try: recipients = request.GET['to'].split(',') url = request.GET['url'] proto, server, path, query, frag = urlsplit(url) if query: path += '?' + query conn = HTTPConnection(server) conn.request('GET',path) resp = conn.getresponse() assert resp.status == 200, "Failed HTTP response %s %s" % (resp.status, resp.reason) rawData = resp.read() conn.close() message = MIMEMultipart() message['Subject'] = "Graphite Image" message['To'] = ', '.join(recipients) message['From'] = 'composer@%s' % gethostname() text = MIMEText( "Image generated by the following graphite URL at %s\r\n\r\n%s" % (ctime(),url) ) image = MIMEImage( rawData ) image.add_header('Content-Disposition', 'attachment', filename="composer_" + strftime("%b%d_%I%M%p.png")) message.attach(text) message.attach(image) s = SMTP(settings.SMTP_SERVER) s.sendmail('composer@%s' % gethostname(),recipients,message.as_string()) s.quit() return HttpResponse( "OK" ) except: return HttpResponse( format_exc() )
class HTTPSpeakerClient: """Emacspeak HTTP speech client, for HTTPSpeaker instances.""" def __init__(self, host="127.0.0.1", port=8000): "Initialize client to connect to server at given host and port." self._connection=HTTPConnection(host, port) def postCommand(self, command, arg=""): """Post command, with argument arg (default, empty), to the speech server. Returns the body of the server's HTTP response, if any. On error, HTTPSpeakerError is raised.""" body = command if arg: body += ": " + arg self._connection.request("POST", "/", body, {"Content-type": "text/plain"}) response=self._connection.getresponse() if response.status != 200: raise HTTPSpeakerError(response.status, response.reason) return response.read() def speak(self, text): "Speak the supplied string." self.postCommand("speak", text) def stop(self): "Stop speaking." self.postCommand("stop") def isSpeaking(self): "Return '0' when not speaking." return self.postCommand("isSpeaking") def close(self): "Close the connection to the speech server." self._connection.close()
class RESTClient(object): """ HTTP client HTTPS client based on the provided URL (http:// or https://) """ def __init__(self, url, cert=None, key=None): logging.info("RESTClient URL: %s" % url) if url.startswith("https://"): logging.info("Using HTTPS protocol, getting user identity files ...") proxy_file = "/tmp/x509up_u%s" % os.getuid() if not os.path.exists(proxy_file): proxy_file = "UNDEFINED" cert_file = cert or os.getenv("X509_USER_CERT", os.getenv("X509_USER_PROXY", proxy_file)) key_file = key or os.getenv("X509_USER_KEY", os.getenv("X509_USER_PROXY", proxy_file)) logging.info("Identity files:\n\tcert file: '%s'\n\tkey file: '%s' " % (cert_file, key_file)) url = url.replace("https://", "") logging.info("Creating connection HTTPS ...") self.conn = HTTPSConnection(url, key_file=key_file, cert_file=cert_file) if url.startswith("http://"): logging.info("Using HTTP protocol, creating HTTP connection ...") url = url.replace("http://", "") self.conn = HTTPConnection(url) def http_request(self, verb, uri, data=None, header=None): logging.debug("Request: %s %s %s ..." % (verb, uri, data)) self.conn.request(verb, uri, body=data, headers=self.headers) resp = self.conn.getresponse() data = resp.read() logging.debug("Status: %s" % resp.status) logging.debug("Reason: %s" % resp.reason) return resp.status, data
def urlopen(url, svprev, formdata): ua = "SPlayer Build %d" % svprev #prepare data #generate a random boundary boundary = "----------------------------" + "%x"%random.getrandbits(48) data = [] for item in formdata: data.append("--" + boundary + "\r\nContent-Disposition: form-data; name=\"" + item[0] + "\"\r\n\r\n" + item[1] + "\r\n") data.append("--" + boundary + "--\r\n") data = "".join(data) cl = str(len(data)) r = urlparse(url) h = HTTPConnection(r.hostname) h.connect() h.putrequest("POST", r.path, skip_host=True, skip_accept_encoding=True) h.putheader("User-Agent", ua) h.putheader("Host", r.hostname) h.putheader("Accept", "*/*") h.putheader("Content-Length", cl) h.putheader("Expect", "100-continue") h.putheader("Content-Type", "multipart/form-data; boundary=" + boundary) h.endheaders() h.send(data) resp = h.getresponse() if resp.status != OK: raise Exception("HTTP response " + str(resp.status) + ": " + resp.reason) return resp
def check_url_path(path, redirected=0): if redirected > MAX_REDIRECTION_ALLOWED: return False try: code = '' parse_result = urlparse(path) server_name = parse_result.netloc urlpath = parse_result.path if not urlpath: # Just a server, as with a repo. with contextlib.closing(urllib2.urlopen(path)) as res: code = res.getcode() else: # socket.gaierror could be raised, # which is a child class of IOError conn = HTTPConnection(server_name, timeout=15) # Don't try to get the whole file: conn.request('HEAD', path) response = conn.getresponse() code = response.status conn.close() if code == 200: return True elif code == 301 or code == 302: for header in response.getheaders(): if header[0] == 'location': return check_url_path(header[1], redirected+1) else: return False except (urllib2.URLError, HTTPException, IOError, ValueError): return False return True
def getresponse(self): r = LibcloudHTTPConnection.getresponse(self) if self.log is not None: r, rv = self._log_response(r) self.log.write(rv + "\n") self.log.flush() return r
def proxy(request): if 'url' not in request.GET: return HttpResponse( "The proxy service requires a URL-encoded URL as a parameter.", status=400, content_type="text/plain" ) url = urlsplit(request.GET['url']) # Don't allow localhost connections unless in DEBUG mode if not settings.DEBUG and re.search('localhost|127.0.0.1', url.hostname): return HttpResponse(status=403) locator = url.path if url.query != "": locator += '?' + url.query if url.fragment != "": locator += '#' + url.fragment # Strip all headers and cookie info headers = {} conn = HTTPConnection(url.hostname, url.port) if url.scheme == "http" else HTTPSConnection(url.hostname, url.port) conn.request(request.method, locator, request.raw_post_data, headers) result = conn.getresponse() response = HttpResponse( valid_response(result.read()), status=result.status, content_type=result.getheader("Content-Type", "text/plain") ) return response
def _select(self, params): # encode the query as utf-8 so urlencode can handle it params['q'] = unicode_safe(params['q']) path = '%s/select/?%s' % (self.path, urlencode(params)) conn = HTTPConnection(self.host, self.port) conn.request('GET', path) return conn.getresponse()
def proxy(request): if 'url' not in request.GET: return HttpResponse( "The proxy service requires a URL-encoded URL as a parameter.", status=400, content_type="text/plain") url = urlsplit(request.GET['url']) locator = url.path if url.query != "": locator += '?' + url.query if url.fragment != "": locator += '#' + url.fragment headers = {} if settings.SESSION_COOKIE_NAME in request.COOKIES: headers["Cookie"] = request.META["HTTP_COOKIE"] if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META: headers["Content-Type"] = request.META["CONTENT_TYPE"] if request.META.get('HTTP_AUTHORIZATION'): headers['AUTHORIZATION'] = request.META.get('HTTP_AUTHORIZATION') conn = HTTPConnection(url.hostname, url.port) conn.request(request.method, locator, request.raw_post_data, headers) result = conn.getresponse() response = HttpResponse(result.read(), status=result.status, content_type=result.getheader( "Content-Type", "text/plain")) if result.getheader('www-authenticate'): response['www-authenticate'] = result.getheader('www-authenticate') return response
def download_api_data(filename, coord, api_base, projection): """ Download API data for a tile to a named file, return size in kilobytes. """ s, host, path, p, q, f = urlparse(api_base) bbox = coordinate_latlon_bbox(coord, projection) path = join(path, 'api/0.6/map?bbox=%.6f,%.6f,%.6f,%.6f' % bbox) conn = HTTPConnection(host) conn.request('GET', path, headers={'Accept-Encoding': 'compress, gzip'}) resp = conn.getresponse() assert resp.status == 200, (resp.status, resp.read()) if resp.getheader('Content-Encoding') == 'gzip': disk = open(filename, 'w') else: raise Exception((host, path)) disk = GzipFile(filename, 'w') bytes = resp.read() disk.write(bytes) disk.close() return len(bytes) / 1024.
def contact_vessel(self, vessel_ip, path, action, key, value): # the Boolean variable we will return success = False # The variables must be encoded in the URL format, through urllib.urlencode post_content = urlencode({ 'action': action, 'key': key, 'value': value }) # the HTTP header must contain the type of data we are transmitting, here URL encoded headers = {"Content-type": "application/x-www-form-urlencoded"} # We should try to catch errors when contacting the vessel try: # We contact vessel:PORT_NUMBER since we all use the same port # We can set a timeout, after which the connection fails if nothing happened connection = HTTPConnection("%s:%d" % (vessel_ip, PORT_NUMBER), timeout=30) # We only use POST to send data (PUT and DELETE not supported) action_type = "POST" # We send the HTTP request connection.request(action_type, path, post_content, headers) # We retrieve the response response = connection.getresponse() # We want to check the status, the body should be empty status = response.status # If we receive a HTTP 200 - OK if status == 200: success = True # We catch every possible exceptions except Exception as e: print "Error while contacting %s" % vessel_ip # printing the error given by Python print(e) # we return if we succeeded or not return success
def GetSesc(self): print "[+] Trying to read Sesc" for i in range(0, 2): conn = HTTPConnection(target, port) conn.request("GET", path + "index.php?action=pm;sa=manlabels;", {}, { "Accept": "text/plain", "Cookie": sn + "=" + sv + ";" }) rsp = conn.getresponse() r = rsp.read() if rsp.status == 404: exit("[-] Error 404. Not Found") elif r.find('<input type="hidden" name="sc" value="') != -1 and r.find( '" />') != -1: self.sesc = r.split( '<input type="hidden" name="sc" value="')[1].split('" />')[0] if len(self.sesc) != 32: exit("[-] Invalid Sesc") print "[+] Sesc has been successfully read ==> " + self.sesc else: exit("[-] Unable to find Sesc")
def notifyNodeServer(result): ret = False notifyPayload = dictToJson(result) log.msg( '[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer parms: ' % (game, tradeNo), notifyPayload) try: nodeHttpServer = urlparse.urlparse( getAppConfig('node.httpserver', orderServerOption.get('env'))) nodeHttpHost = nodeHttpServer.hostname nodeHttpPort = nodeHttpServer.port httpClient = HTTPConnection(host=nodeHttpHost, port=nodeHttpPort) endPoint = "/api/games/reCharge" httpClient.request( 'POST', endPoint, notifyPayload, headers={'Content-Type': 'application/json'}) response = httpClient.getresponse() log.msg( '[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer : ' % (game, tradeNo), response.status, response.reason) log.msg( '[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer : ' % (game, tradeNo), response.read()) httpClient.close() ret = True except Exception, e: log.msg( '[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer fail: ' % (game, tradeNo), e.messag) log.msg( '[consumerService, %s] %s: orderProcess:orderProcessComplete:notifyNodeServer trace: ' % (game, tradeNo), e)
def s3save(url, data): auth = "LOW %s:%s" % (appauth.key, appauth.secret) c = HTTPConnection("s3.us.archive.org") c.request("PUT", url, data, {"Authorization": auth}) r = c.getresponse()
def fproxyGet(self, path): """ Fetches from fproxy, returns (status, mimetype, data) """ server = self.server headers = self.headers print "--------------------------------------------" print "** path=%s" % path # first scenario - user is pointing their browser directly at # fproxyfproxy, barf! if not path.startswith("http://"): self.send_response(400) self.send_header("Content-type", "text/html") data = "\n".join([ "<html><head>", "<title>Access Denied</title>", "</head><body>", "<h1>Access Denied</h1>", "Sorry, but FProxyProxy is an http proxy server.<br>", "Please don't try to access it like a web server.", "</body></html>", "", ]) self.send_header("Content-Length", str(len(data))) self.send_header("Location", location) self.end_headers() self.wfile.write(data) self.wfile.flush() return # convert path to relative path = "/" + path[7:].split("/", 1)[-1] #print "** path=%s" % repr(path) try: # check host header hostname = headers.get("Host", 'fproxy') pathbits = path.split("/") print "** hostname = %s" % hostname # second scenario, user has just given a domain name without trailing / if len(pathbits) == 1: # redirect to force trailing slash location = path + "/" print "** redirecting to: %s" % location self.send_response(301) self.send_header("Content-type", "text/html") data = "\n".join([ "<html><head>", "<title>Permanent redirect: new URI</title>", "</head><body>", "<h1>Permanent redirect: new URI</h1>", "<a href=\"%s\">Click here</a>", "</body></html>", "", ]) % location self.send_header("Content-Length", str(len(data))) self.send_header("Location", location) self.end_headers() self.wfile.write(data) self.wfile.flush() return tail = "/".join(pathbits[1:]) # third scenario - request into fproxy if hostname == 'fproxy': # tis an fproxy request, go straight through conn = HTTPConnection(server.fproxyHost, server.fproxyPort) conn.request("GET", path) resp = conn.getresponse() self.send_response(resp.status) self.send_header("Content-type", resp.getheader("Content-Type", "text/plain")) data = resp.read() self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) self.wfile.flush() conn.close() return else: # final scenario - some other domain, try lookup uri = server.node.namesiteLookup(hostname) if not uri: # lookup failed, do the usual 404 thang print "** lookup of domain %s failed" % hostname self.send_response(404) self.send_header("Content-type", "text/html") data = "\n".join([ "<html><head>", "<title>404 - Freenet name not found</title>", "</head><body", "<h1>404 - Freenet name not found</title>", "The pyFreenet name service was unable to resolve ", "the name %s" % hostname, "<br><br>", "You might like to find its freenet uri and try that ", "within <a href=\"/fproxy/\">FProxy</a>", "</body></html>", "", ]) self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) self.wfile.flush() # lookup succeeded - ok to go now via fproxy conn = HTTPConnection(server.fproxyHost, server.fproxyPort) newpath = "/" + uri if tail: if not newpath.endswith("/"): newpath += "/" newpath += tail print "** newpath=%s" % newpath conn.request("GET", newpath) resp = conn.getresponse() print "** status=%s" % resp.status self.send_response(resp.status) self.send_header("Content-type", resp.getheader("Content-Type", "text/plain")) # has fproxy sent us a redirect? if resp.status == 301: # yuck, fproxy is telling us to redirect, which # sadly means we have to lose the domain name # from our browser address bar location = resp.getheader("location") newLocation = "http://fproxy" + location print "*** redirected!!!" print "*** old location = %s" % location print "*** --> %s" % newLocation self.send_header("Location", newLocation) # get the data from fproxy and send it up to the client data = resp.read() self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) self.wfile.flush() conn.close() return return except socket.error: raise
def _head(cls, reference): conn = HTTPConnection(str(reference.authority)) # XXX Add the query conn.request('HEAD', str(reference.path)) return conn.getresponse()
class GoogleSuggestions(): def __init__(self): self.hl = "en" self.conn = None def prepareQuery(self): #GET /complete/search?output=toolbar&client=youtube-psuggest&xml=true&ds=yt&hl=en&jsonp=self.gotSuggestions&q=s #self.prepQuerry = "/complete/search?output=toolbar&client=youtube&xml=true&ds=yt&" self.prepQuerry = "/complete/search?output=chrome&client=chrome&" if self.hl is not None: self.prepQuerry = self.prepQuerry + "hl=" + self.hl + "&" self.prepQuerry = self.prepQuerry + "jsonp=self.gotSuggestions&q=" print("[MyTube - GoogleSuggestions] prepareQuery:", self.prepQuerry) def getSuggestions(self, queryString): self.prepareQuery() if queryString is not "": query = self.prepQuerry + quote(queryString) self.conn = HTTPConnection("google.com") try: self.conn = HTTPConnection("google.com") self.conn.request("GET", query, "", {"Accept-Encoding": "UTF-8"}) except (CannotSendRequest, gaierror, error): self.conn.close() print( "[MyTube - GoogleSuggestions] Can not send request for suggestions" ) return None else: try: response = self.conn.getresponse() except BadStatusLine: self.conn.close() print( "[MyTube - GoogleSuggestions] Can not get a response from google" ) return None else: if response.status == 200: data = response.read() header = response.getheader( "Content-Type", "text/xml; charset=ISO-8859-1") charset = "ISO-8859-1" try: charset = header.split(";")[1].split("=")[1] print( "[MyTube - GoogleSuggestions] Got charset %s" % charset) except: print( "[MyTube - GoogleSuggestions] No charset in Header, falling back to %s" % charset) data = data.decode(charset).encode("utf-8") self.conn.close() return data else: self.conn.close() return None else: return None
class TestRegression(TestCase): def setUp(self): self.ip = "localhost" self.port = "8081" self.headers = {'Content-Type': 'application/json', 'Accept': 'application/json'} self.timeout = 10 # connect self.conn = HTTPConnection(self.ip, self.port, timeout=self.timeout) def tearDown(self): self.conn.close() def test_1_basic_request(self): """ """ # request an pa computation path = "/5gt/so/v1/PAComp" data = {"ReqId": "string", "nfvi": { "NFVIPoPs": [ { "Id": "DC1", "gw_ip_address": "172.16.102.116", "capabilities": { "cpu": 3, "ram": 2400, "storage": 135000 }, "availableCapabilities": { "cpu": 78, "ram": 2400, "storage": 135000 }, "internal_latency": 1.04 }, { "Id": "DC2", "gw_ip_address": "172.16.102.117", "capabilities": { "cpu": 3, "ram": 160, "storage": 7000 }, "availableCapabilities": { "cpu": 85, "ram": 160, "storage": 7000 }, "internal_latency": 4.16 }, { "Id": "DC3", "gw_ip_address": "172.16.102.118", "capabilities": { "cpu": 2, "ram": 160, "storage": 7000 }, "availableCapabilities": { "cpu": 40, "ram": 160, "storage": 7000 }, "internal_latency": 4.16 } ], "LLs": [ { "LLid": "LL1", "capacity": { "total": 125000000000, "available": 1250000000 }, "delay": 3.3, "length": 250, "source": { "Id": "DC1", "GwIpAddress": "172.16.102.116" }, "destination": { "Id": "DC2", "GwIpAddress": "172.16.102.117" } }, { "LLid": "LL2", "capacity": { "total": 1250000000000, "available": 1250000000 }, "delay": 1.3, "length": 150, "source": { "Id": "DC1", "GwIpAddress": "172.16.102.116" }, "destination": { "Id": "DC3", "GwIpAddress": "172.16.102.118" } } ] }, "nsd": { "Id": "Req1", "name": "req1", "VNFs": [ { "VNFid": "VNFa", "instances": 1, "requirements": { "cpu": 3, "ram": 8, "storage": 1 } }, { "VNFid": "VNFb", "instances": 2, "requirements": { "cpu": 4, "ram": 5, "storage": 2 } } ], "VNFLinks": [ { "source": "172.16.102.116", "required_capacity": 1250000 } ], "max_latency": 9.6 }, "callback": "string" } self.conn.request("POST", path, dumps(data), self.headers) self.conn.sock.settimeout(self.timeout) rsp = self.conn.getresponse() data = rsp.read() self.assertEqual(rsp.status, 201)
def run(self): headers = { 'Ntrip-Version': 'Ntrip/2.0', 'User-Agent': 'NTRIP ntrip_ros', 'Connection': 'close', 'Authorization': 'Basic ' + b64encode(self.ntc.ntrip_user + ':' + str(self.ntc.ntrip_pass)) } connection = HTTPConnection(self.ntc.ntrip_server) connection.request('GET', '/' + self.ntc.ntrip_stream, self.ntc.nmea_gga, headers) response = connection.getresponse() if response.status != 200: raise Exception("Error: Response not HTTP200 (OK)") buf = "" rmsg = Message() restart_count = 0 while not self.stop: ''' data = response.read(100) pos = data.find('\r\n') if pos != -1: rmsg.message = buf + data[:pos] rmsg.header.seq += 1 rmsg.header.stamp = rospy.get_rostime() buf = data[pos+2:] self.ntc.pub.publish(rmsg) else: buf += data ''' ''' This now separates individual RTCM messages and publishes each one on the same topic ''' data = response.read(1) if len(data) != 0: if ord(data[0]) == 211: buf += data data = response.read(2) buf += data cnt = ord(data[0]) * 256 + ord(data[1]) data = response.read(2) buf += data typ = (ord(data[0]) * 256 + ord(data[1])) / 16 print(str(datetime.now()), cnt, typ) cnt = cnt + 1 for x in range(cnt): data = response.read(1) buf += data rmsg.message = buf rmsg.header.seq += 1 rmsg.header.stamp = rospy.get_rostime() self.ntc.pub.publish(rmsg) buf = "" else: print(data) else: ''' If zero length data, close connection and reopen it ''' restart_count = restart_count + 1 print("Zero length ", restart_count) connection.close() connection = HTTPConnection(self.ntc.ntrip_server) connection.request('GET', '/' + self.ntc.ntrip_stream, self.ntc.nmea_gga, headers) response = connection.getresponse() if response.status != 200: raise Exception("Error: Response not HTTP200 (OK)") buf = "" connection.close()
class HTTPRelayClient: def __init__(self, target): # Target comes as protocol://target:port/path self.target = target proto, host, path = target.split(':') host = host[2:] self.path = '/' + path.split('/', 1)[1] if proto.lower() == 'https': #Create unverified (insecure) context try: uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) self.session = HTTPSConnection(host, context=uv_context) except AttributeError: #This does not exist on python < 2.7.11 self.session = HTTPSConnection(host) else: self.session = HTTPConnection(host) self.lastresult = None def sendNegotiate(self, negotiateMessage): #Check if server wants auth self.session.request('GET', self.path) res = self.session.getresponse() res.read() if res.status != 401: logging.info( 'Status code returned: %d. Authentication does not seem required for URL' % res.status) try: if 'NTLM' not in res.getheader('WWW-Authenticate'): logging.error( 'NTLM Auth not offered by URL, offered protocols: %s' % res.getheader('WWW-Authenticate')) return False except (KeyError, TypeError): logging.error( 'No authentication requested by the server for url %s' % self.target) return False #Negotiate auth negotiate = base64.b64encode(negotiateMessage) headers = {'Authorization': 'NTLM %s' % negotiate} self.session.request('GET', self.path, headers=headers) res = self.session.getresponse() res.read() try: serverChallengeBase64 = re.search( 'NTLM ([a-zA-Z0-9+/]+={0,2})', res.getheader('WWW-Authenticate')).group(1) serverChallenge = base64.b64decode(serverChallengeBase64) return serverChallenge except (IndexError, KeyError, AttributeError): logging.error('No NTLM challenge returned from server') def sendAuth(self, authenticateMessageBlob, serverChallenge=None): #Negotiate auth auth = base64.b64encode(authenticateMessageBlob) headers = {'Authorization': 'NTLM %s' % auth} self.session.request('GET', self.path, headers=headers) res = self.session.getresponse() if res.status == 401: return False else: logging.info( 'HTTP server returned error code %d, treating as a succesful login' % res.status) #Cache this self.lastresult = res.read() return True #SMB Relay server needs this @staticmethod def get_encryption_key(): return None
def test_redirects(self): ''' Check a selection of HTTP redirect pairs. ''' pairs = [ ('/accelerator', '/companies/accelerator-faq/'), ('/incubator', '/companies/incubator-faq/'), ('/projects', '/apps/'), ('/brigade/projects', '/brigade/projects'), ('/focus', '/our-work/focus-areas/'), ('/governments/capabilities', '/governments/principles/'), ('/governments/capabilities/index.html', '/governments/principles/index.html'), ('/governments/capabilities/open-data', '/governments/principles/open-data/'), ('/procurement', '/governments/principles/procurement/'), ('/governments/capabilities/procurement', '/governments/principles/procurement/'), ('/team', '/about/team/'), ('/projects', '/apps/'), ('/fellows/job-description', '/geeks/fellowship-faq/'), ('/fellows/apply', '/geeks/fellowship-apply/'), ('/fellows/alumni-fellows', '/geeks/our-geeks/alumni-fellows/'), ('/alumni-fellows', '/geeks/our-geeks/alumni-fellows/'), ('/fellows/2011-fellows', '/geeks/our-geeks/2011-fellows/'), ('/2011-fellows', '/geeks/our-geeks/2011-fellows/'), ('/fellows/2012-fellows', '/geeks/our-geeks/2012-fellows/'), ('/2012-fellows', '/geeks/our-geeks/2012-fellows/'), ('/fellows/2013-fellows', '/geeks/our-geeks/2013-fellows/'), ('/2013-fellows', '/geeks/our-geeks/2013-fellows/'), ('/fellows/2014-fellows', '/geeks/our-geeks/2014-fellows/'), ('/2014-fellows', '/geeks/our-geeks/2014-fellows/'), ('/startups', '/about/companies/'), ('/fellowship', '/about/fellowship/'), # ('/fellows', '/about/fellowship/'), ('/procurement', '/governments/principles/procurement/'), ('/austin', '/governments/austin/'), ('/honolulu', '/governments/honolulu/'), ('/alex-pandel', '/people/alex-pandel/'), ('/alex-yule', '/people/alex-yule/'), ('/cities', '/governments/'), ('/cities/2015-partners', '/governments/2015-partners/'), ('/codeacross-2014', '/events/codeacross-2014/'), ('/codeacross', '/events/codeacross-2015/'), ('/02-18-2014', '/peer-network-training/02-18-2014/'), ('/brigade12-12-2013', '/brigade-training/brigade12-12-2013/'), ('/09-19-2013', '/brigade-training/09-19-2013/'), ('/blog/meettheauthors', '/peer-network-training/meettheauthors/'), ('/09-24-2013', '/peer-network-training/09-24-2013/'), ('/blog/08-22-2013', '/peer-network-training/08-22-2013/'), ('/08-07-2013', '/peer-network-training/08-07-2013/'), ('/blog/ask-a-fellow-2013', '/peer-network-training/ask-a-fellow-2013/'), ('/accelerator', '/companies/accelerator-faq/'), ('/accelerator-2013', '/companies/accelerator-faq/'), ('/accelerator-2', '/companies/accelerator-faq/'), ('/startups/accelerator', '/companies/accelerator-faq/'), ('/incubator', '/companies/incubator-faq/'), ('/incubator-2', '/companies/incubator-faq/'), ('/startups/incubator-2', '/companies/incubator-faq/'), ('/who-we-are', '/about/team/'), ('/donors', '/supporters/'), ('/how-to-help', '/geeks/'), ('/code-across-america', '/events/codeacross-2012/'), ('/fellows/faq', '/geeks/fellowship-faq/'), ('/fellows/training', '/geeks/fellowship-faq/'), ('/fellows/institute', '/geeks/fellowship-faq/'), ('/governments/expectations-of-our-city-partners', '/governments/fellowship/'), ('/2012-city-videos', '/communications/testimonial/'), ('/geeks/fellows/alumni-fellows', '/geeks/our-geeks/alumni-fellows/'), ('/fellows/current-fellows', '/geeks/our-geeks/2013-fellows/'), ('/fellows/alumni-fellows/2013-fellows', '/geeks/our-geeks/2013-fellows/'), ('/geeks/our-startups', '/companies/our-companies/'), ('/geeks/accelerator-faq', '/companies/accelerator-faq/'), ('/geeks/accelerator-apply', '/companies/accelerator-apply/'), ('/geeks/fellowship', '/about/fellowship/'), ('/governments/city-impact', '/about/fellowship/'), ('/the-program', '/about/fellowship/'), # ('/about/fellowship', '/governments/fellowship/'), ('/governments/fellowship-faq', '/governments/fellowship/'), ('/governments/2013-apply-now', '/governments/fellowship/'), ('/governments/apply', '/forms/governments/fellowship/apply/'), ('/governments/apply-now', '/forms/governments/fellowship/apply/'), ('/governments/fellowship-apply', '/forms/governments/fellowship/apply/'), ('/forms/governments/fellowship/interest', '/forms/governments/fellowship/apply/'), ('/city-alumni', '/governments/alumni/'), ('/cfa-2012', '/governments/alumni/'), ('/2012', '/governments/alumni/'), ('/governments/faqs-cities-applying-to-program', '/governments/fellowship/'), ('/civic-data-standards', '/governments/data-standards-faq/'), ('/application-info', '/governments/fellowship/'), ('/application-questions', '/governments/fellowship/'), ('/governments/call-for-applications', '/governments/fellowship/'), ('/2013-partners', '/governments/alumni/'), ('/city-current', '/governments/alumni/'), ('/governments/current', '/governments/alumni/'), ('/2013-partners/kansas-city', '/governments/kansascity/'), ('/2013-partners/louisville', '/governments/louisville/'), ('/2013-partners/las-vegas', '/governments/lasvegas/'), ('/city-current/kansas-city', '/governments/kansascity/'), ('/city-current/louisville', '/governments/louisville/'), ('/city-current/las-vegas', '/governments/lasvegas/'), ('/2012-partners/austin', '/governments/austin/'), ('/2012-partners/chicago', '/governments/chicago/'), ('/2012-partners/detroit', '/governments/detroit/'), # ('/2012-city-finalists', '/2012-city-finalists/'), ('/2012-city-finalists/chicago', '/governments/chicago/'), ('/2012-city-finalists/detroit', '/governments/detroit/'), ('/city-alumni/boston', '/governments/boston/'), ('/city-alumni/chicago', '/governments/chicago/'), ('/city-alumni/philadelphia', '/governments/philadelphia/'), ('/governments/summit-county', '/governments/summitcounty/'), ('/focus', '/our-work/focus-areas/'), ('/projects/new-york-city-hhc-accelerator/', '/projects/new-york-city-hhs-accelerator/'), # # This actually goes to BSD, but we check for it anyway. # ('/donate', '/page/contribute/default'), ] for (start_path, end_path) in pairs: url = 'http://0.0.0.0:{0}{1}'.format(self.port, start_path) for i in range(100): if i > 10: raise Exception( 'Too many redirects from {0}, now at {1}'.format( start_path, url_path)) _, url_host, url_path, _, _, _ = urlparse(url) conn = HTTPConnection(*url_host.split(':')) conn.request('GET', url_path) resp = conn.getresponse() conn.close() if resp.status not in range(300, 399): break url = urljoin(url, resp.getheader('location')) _, new_host, url_path, _, _, _ = urlparse(url) # Stop if we go off-site if new_host != url_host: break assert end_path == url_path, '{0} instead of {1} from {2}'.format( url_path, end_path, start_path)
if slug not in files: files[slug] = dict() files[slug][key] = slug_file coast['coastline-good'] = {} for prj in ('merc', 'latlon'): file = 'coastline-good-%s.tar.bz2' % prj href = urljoin(base_url, file) url = urlparse(href) conn = HTTPConnection(url.netloc) conn.request('HEAD', url.path) resp = conn.getresponse() size = resp.getheader('content-length') date = parser.parse(resp.getheader('last-modified')) date = '%s %d, %s' % (months[date.month], date.day, date.year) coast['coastline-good'][prj] = (file, int(size), href, date) cities = list(DictReader(open('cities.txt'), dialect='excel-tab')) # print >> index, """<!DOCTYPE html> <html lang="en"> <head> <title>Metro Extracts</title> <meta http-equiv="content-type" content="text/html; charset=utf-8">
def makeRequest(url, values=None, verb='GET', accept="text/plain", contentType=None, secure=False, secureParam={}): headers = {} contentType = contentType or "application/x-www-form-urlencoded" headers = { "content-type": contentType, "Accept": accept, "cms-auth-status": "NONE" } if secure: headers.update({ "cms-auth-status": "OK", "cms-authn-dn": "/DC=ch/OU=Organic Units/OU=Users/CN=Fake User", "cms-authn-name": "Fake User", "cms-authz-%s" % secureParam['role']: "group:%s site:%s" % (secureParam['group'], secureParam['site']) }) headers["cms-authn-hmac"] = _generateHash(secureParam["key"], headers) data = None if verb == 'GET' and values: data = urllib.urlencode(values, doseq=True) elif verb != 'GET' and values: # needs to test other encoding type if contentType == "application/x-www-form-urlencoded": data = urllib.urlencode(values) else: # for other encoding scheme values assumed to be encoded already data = values parser = urlparse(url) uri = parser.path if parser.query: uri += "?" + parser.query if verb == 'GET' and data != None: uri = '%s?%s' % (uri, data) # need to specify Content-length for POST method # TODO: this function needs refactoring - too verb-related branching if verb != 'GET': if data: headers.update({"content-length": len(data)}) else: headers.update({"content-length": 0}) conn = HTTPConnection(parser.netloc) conn.connect() conn.request(verb, uri, data, headers) response = conn.getresponse() data = response.read() conn.close() cType = response.getheader('content-type').split(';')[0] return data, response.status, cType, response
def sendData(uri, method="POST", data_raw=None, data_enc=None, svar_fra_mottager=1, tag='karusell', reqursivLevel=1, multipefaktor=1): """Sender gluoner til de forskjellige transmitterne. Denne brukes av trådmodulen og blir kjørt en gang pr. transmitter""" ret_svar = {} utverdier = {} #Lager midlertidig svar som ev. oversktives senere #Lock og skriv ret_svar['status'] = '0' ret_svar['reason'] = 'Starter utsendelse' utverdier[tag] = ret_svar #start = time.time() #Sett opp headers if data_raw: headers = { "Content-type": "application/xml", "Accept": "*/*", "User-Agent": "Gluon 0.5.0 [no] (%s; U)" % sys.platform } data = data_raw else: headers = { "Content-type": "application/x-www-form-urlencoded", "Accept": "*/*", "User-Agent": "Gluon 0.5.0 [no] (%s; U)" % sys.platform } data = data_enc #Dele opp uri til hostname og url host, url = uri[7:].split('/', 1) start = time.time() try: conn = HTTPConnection(host) conn.request(method, '/' + url, data, headers) ret_svar['status'] = '3' ret_svar[ 'reason'] = 'Sendt data - transmitter svarer ikke. Ting er sansynlighvis OK' #Dette skyldes som regel en treg mottaker og er ikke nødvendighvis en feil time.sleep(0.003) except: #Legge inn forskjellige verdier her #Ev. legge inn rutine for automatisk forsøk igjen ret_svar['status'] = '1' ret_svar['reason'] = 'Kunne ikke lage forbindelse' else: if svar_fra_mottager: svar = conn.getresponse() #print dir(svar) ret_svar['status'] = svar.status ret_svar['reason'] = svar.reason if method == 'HEAD': ret_svar['msg'] = '' else: ret_svar['msg'] = svar.read() ret_svar['duration'] = "PT%sS" % (time.time() - start) else: tid = time.time() - start ret_svar['msg'] = 'TID: %s HVEM: %s' % (tid, tag) conn.close() #Sjekk om vi har en 302 if ret_svar['status'] == 302: #Vi sender videre i denne omgang if reqursivLevel < maxReqursivLevel: nyUrl = svar.getheader('Location') nytransmitter = tag + ":redirekt_%s" % reqursivLevel sendData(nyUrl, data_raw=data_raw, svar_fra_mottager=svar_fra_mottager, tag=nytransmitter, reqursivLevel=reqursivLevel + 1) #ret_svar['msg'] = nyUrl #Lock og skriv utverdier[tag] = ret_svar return utverdier
try: conf = open("./portconf", "r") pport = conf.readline().rstrip().split(':')[1] sport1 = conf.readline().rstrip().split(':')[1] sport2 = conf.readline().rstrip().split(':')[1] server1 = ServerThread(int(sport1)) server1.start() dataFile = open('basic', "r") cdata = dataFile.read() r = False proxy = '127.0.0.1:' + pport conn = HTTPConnection(proxy) conn.request("GET", "http://127.0.0.1:" + sport1 + "/basic") resp = conn.getresponse() data = resp.read() conn.close() time.sleep(3) conn2 = HTTPConnection(proxy) conn2.request("GET", "http://127.0.0.1:" + sport1 + "/basic") resp2 = conn2.getresponse() data2 = resp2.read() conn2.close() time.sleep(3) conn3 = HTTPConnection(proxy) conn3.request("GET", "http://127.0.0.1:" + sport1 + "/basic") resp3 = conn3.getresponse() data3 = resp3.read()
class SingleDownload(SingleDownloadHelperInterface): def __init__(self, downloader, url): SingleDownloadHelperInterface.__init__(self) self.downloader = downloader self.baseurl = url try: self.scheme, self.netloc, path, pars, query, fragment = urlparse(url) except: self.downloader.errorfunc('cannot parse http seed address: ' + url) return if self.scheme != 'http': self.downloader.errorfunc('http seed url not http: ' + url) return self.proxyhost = find_proxy(url) try: if self.proxyhost is None: self.connection = HTTPConnection(self.netloc) else: self.connection = HTTPConnection(self.proxyhost) except: self.downloader.errorfunc('cannot connect to http seed: ' + url) return self.seedurl = path if pars: self.seedurl += ';' + pars self.seedurl += '?' if query: self.seedurl += query + '&' self.seedurl += 'info_hash=' + urllib.quote(self.downloader.infohash) self.measure = Measure(downloader.max_rate_period) self.index = None self.url = '' self.requests = [] self.request_size = 0 self.endflag = False self.error = None self.retry_period = 30 self._retry_period = None self.errorcount = 0 self.goodseed = False self.active = False self.cancelled = False self.resched(randint(2, 10)) def resched(self, len = None): if len is None: len = self.retry_period if self.errorcount > 3: len = len * (self.errorcount - 2) self.downloader.rawserver.add_task(self.download, len) def _want(self, index): if self.endflag: return self.downloader.storage.do_I_have_requests(index) else: return self.downloader.storage.is_unstarted(index) def download(self): if DEBUG: print 'http-sdownload: download()' if self.is_frozen_by_helper(): if DEBUG: print 'http-sdownload: blocked, rescheduling' self.resched(1) return self.cancelled = False if self.downloader.picker.am_I_complete(): self.downloader.downloads.remove(self) return self.index = self.downloader.picker.next(haveall, self._want, self) if self.index is None and self.frozen_by_helper: self.resched(0.01) return if self.index is None and not self.endflag and not self.downloader.peerdownloader.has_downloaders(): self.endflag = True self.index = self.downloader.picker.next(haveall, self._want, self) if self.index is None: self.endflag = True self.resched() else: self.url = self.seedurl + '&piece=' + str(self.index) self._get_requests() if self.request_size < self.downloader.storage._piecelen(self.index): self.url += '&ranges=' + self._request_ranges() rq = Thread(target=self._request) rq.setName('HoffmanHTTPDownloader' + rq.getName()) rq.setDaemon(True) rq.start() self.active = True def _request(self): import encodings.ascii import encodings.punycode import encodings.idna self.error = None self.received_data = None try: self.connection.request('GET', self.url, None, {'User-Agent': VERSION}) r = self.connection.getresponse() self.connection_status = r.status self.received_data = r.read() except Exception as e: log_exc() self.error = 'error accessing http seed: ' + str(e) try: self.connection.close() except: pass try: self.connection = HTTPConnection(self.netloc) except: self.connection = None self.downloader.rawserver.add_task(self.request_finished) def request_finished(self): self.active = False if self.error is not None: if self.goodseed: self.downloader.errorfunc(self.error) self.errorcount += 1 if self.received_data: self.errorcount = 0 if not self._got_data(): self.received_data = None if not self.received_data: self._release_requests() self.downloader.peerdownloader.piece_flunked(self.index) if self._retry_period: self.resched(self._retry_period) self._retry_period = None return self.resched() def _got_data(self): if self.connection_status == 503: try: self.retry_period = max(int(self.received_data), 5) except: pass return False if self.connection_status != 200: self.errorcount += 1 return False self._retry_period = 1 if len(self.received_data) != self.request_size: if self.goodseed: self.downloader.errorfunc('corrupt data from http seed - redownloading') return False self.measure.update_rate(len(self.received_data)) self.downloader.measurefunc(len(self.received_data)) if self.cancelled: return False if not self._fulfill_requests(): return False if not self.goodseed: self.goodseed = True self.downloader.seedsfound += 1 if self.downloader.storage.do_I_have(self.index): self.downloader.picker.complete(self.index) self.downloader.peerdownloader.check_complete(self.index) self.downloader.gotpiecefunc(self.index) return True def _get_requests(self): self.requests = [] self.request_size = 0L while self.downloader.storage.do_I_have_requests(self.index): r = self.downloader.storage.new_request(self.index) self.requests.append(r) self.request_size += r[1] self.requests.sort() def _fulfill_requests(self): start = 0L success = True while self.requests: begin, length = self.requests.pop(0) if not self.downloader.storage.piece_came_in(self.index, begin, [], self.received_data[start:start + length], length): success = False break start += length return success def _release_requests(self): for begin, length in self.requests: self.downloader.storage.request_lost(self.index, begin, length) self.requests = [] def _request_ranges(self): s = '' begin, length = self.requests[0] for begin1, length1 in self.requests[1:]: if begin + length == begin1: length += length1 continue else: if s: s += ',' s += str(begin) + '-' + str(begin + length - 1) begin, length = begin1, length1 if s: s += ',' s += str(begin) + '-' + str(begin + length - 1) return s def helper_forces_unchoke(self): pass def helper_set_freezing(self, val): self.frozen_by_helper = val
class HTTPRelayClient: def __init__(self, target): # Target comes as protocol://target:port/path self.target = target _, host, path = target.split(':') host = host[2:] self.path = '/' + path.split('/')[1] self.session = HTTPConnection(host) self.lastresult = None def sendNegotiate(self, negotiateMessage): #Check if server wants auth self.session.request('GET', self.path) res = self.session.getresponse() res.read() if res.status != 401: logging.info( 'Status code returned: %d. Authentication does not seem required for URL' % res.status) try: if 'NTLM' not in res.getheader('WWW-Authenticate'): logging.error( 'NTLM Auth not offered by URL, offered protocols: %s' % res.getheader('WWW-Authenticate')) return False except KeyError: logging.error( 'No authentication requested by the server for url %s' % self.target) return False #Negotiate auth negotiate = base64.b64encode(negotiateMessage) headers = {'Authorization': 'NTLM %s' % negotiate} self.session.request('GET', self.path, headers=headers) res = self.session.getresponse() res.read() try: serverChallenge = base64.b64decode( res.getheader('WWW-Authenticate')[5:]) return serverChallenge except (IndexError, KeyError): logging.error('No NTLM challenge returned from server') def sendAuth(self, authenticateMessageBlob, serverChallenge=None): #Negotiate auth auth = base64.b64encode(authenticateMessageBlob) headers = {'Authorization': 'NTLM %s' % auth} self.session.request('GET', self.path, headers=headers) res = self.session.getresponse() if res.status == 401: return False else: logging.info( 'HTTP server returned error code %d, treating as a succesful login' % res.status) #Cache this self.lastresult = res.read() return True #SMB Relay server needs this @staticmethod def get_encryption_key(): return None
class TestRegression(TestCase): def setUp(self): self.ip = "localhost" self.port = "6161" self.headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' } self.timeout = 10 # connect self.conn = HTTPConnection(self.ip, self.port, timeout=self.timeout) def tearDown(self): self.conn.close() def test_1_basic_request(self): """ """ # request an pa computation path = "/5gt/so/v1/PAComp" data = { "ReqId": "string", "nfvi": { "resource_types": ["cpu", "ram", "storage", "bandwidth"], "NFVIPoPs": [{ "id": "A", "location": { "center": { "longitude": 10.878099, "latitude": 1.9921979 }, "radius": 0 }, "gw_ip_address": "10.10.10.1", "capabilities": { "cpu": 100, "ram": 100, "storage": 100, "bandwidth": 100, "mac": True }, "availableCapabilities": { "cpu": 100, "ram": 100, "storage": 100, "bandwidth": 100 }, "failure_rate": 0, "internal_latency": 2.5 }, { "id": "B", "location": { "center": { "longitude": 4.878099, "latitude": 31.9921979 }, "radius": 0 }, "gw_ip_address": "10.10.10.3", "capabilities": { "cpu": 100, "ram": 100, "storage": 100, "bandwidth": 100, "mec": True }, "availableCapabilities": { "cpu": 100, "ram": 100, "storage": 100, "bandwidth": 100 }, "failure_rate": 0, "internal_latency": 3.3 }, { "id": "C", "location": { "center": { "longitude": -3.878099, "latitude": 44.9921979 }, "radius": 0 }, "gw_ip_address": "10.10.10.5", "capabilities": { "cpu": 100, "ram": 100, "storage": 100, "bandwidth": 100 }, "availableCapabilities": { "cpu": 100, "ram": 100, "storage": 100, "bandwidth": 100 }, "failure_rate": 0, "internal_latency": 4.04 }, { "id": "D", "location": { "center": { "longitude": 5.1426034, "latitude": 22.0306169 }, "radius": 0 }, "gw_ip_address": "10.10.10.7", "capabilities": { "cpu": 100, "ram": 100, "storage": 100, "bandwidth": 100 }, "availableCapabilities": { "cpu": 100, "ram": 100, "storage": 100, "bandwidth": 100 }, "failure_rate": 0, "internal_latency": 2.3 }], "LLs": [{ "LLid": "A-B-1", "capacity": { "total": 40, "available": 40 }, "delay": 9, "source": { "id": "A", "GwIpAddress": "10.10.10.1" }, "destination": { "id": "B", "GwIpAddress": "10.10.10.3" } }, { "LLid": "A-B-2", "capacity": { "total": 80, "available": 80 }, "delay": 5, "source": { "id": "A", "GwIpAddress": "10.10.10.1" }, "destination": { "id": "B", "GwIpAddress": "10.10.10.3" } }, { "LLid": "A-D-1", "capacity": { "total": 50, "available": 50 }, "delay": 12, "source": { "id": "A", "GwIpAddress": "10.10.10.1" }, "destination": { "id": "D", "GwIpAddress": "10.10.10.7" } }, { "LLid": "C-D-1", "capacity": { "total": 50, "available": 50 }, "delay": 11, "source": { "id": "C", "GwIpAddress": "10.10.10.5" }, "destination": { "id": "D", "GwIpAddress": "10.10.10.7" } }, { "LLid": "C-A-1", "capacity": { "total": 35, "available": 55 }, "delay": 3, "source": { "id": "C", "GwIpAddress": "10.10.10.5" }, "destination": { "id": "A", "GwIpAddress": "10.10.10.1" } }, { "LLid": "B-A-1", "capacity": { "total": 15, "available": 45 }, "delay": 12, "source": { "id": "B", "GwIpAddress": "10.10.10.3" }, "destination": { "id": "A", "GwIpAddress": "10.10.10.1" } }, { "LLid": "C-B-1", "capacity": { "total": 20, "available": 20 }, "delay": 5, "source": { "id": "C", "GwIpAddress": "10.10.10.5" }, "destination": { "id": "B", "GwIpAddress": "10.10.10.3" } }, { "LLid": "B-C-1", "capacity": { "total": 10, "available": 60 }, "delay": 7, "source": { "id": "B", "GwIpAddress": "10.10.10.3" }, "destination": { "id": "C", "GwIpAddress": "10.10.10.5" } }, { "LLid": "B-D-1", "capacity": { "total": 10, "available": 70 }, "delay": 7, "source": { "id": "B", "GwIpAddress": "10.10.10.3" }, "destination": { "id": "D", "GwIpAddress": "10.10.10.7" } }, { "LLid": "D-C-1", "capacity": { "total": 50, "available": 50 }, "delay": 7, "source": { "id": "D", "GwIpAddress": "10.10.10.7" }, "destination": { "id": "C", "GwIpAddress": "10.10.10.5" } }, { "LLid": "D-A-1", "capacity": { "total": 50, "available": 50 }, "delay": 7, "source": { "id": "D", "GwIpAddress": "10.10.10.7" }, "destination": { "id": "A", "GwIpAddress": "10.10.10.1" } }, { "LLid": "D-B-1", "capacity": { "total": 45, "available": 45 }, "delay": 3, "source": { "id": "D", "GwIpAddress": "10.10.10.7" }, "destination": { "id": "B", "GwIpAddress": "10.10.10.3" } }] }, "nsd": { "id": "rel1-nsd-test", "name": "multi-graph-ns", "SAP": [{ "CPid": "CP1", "location": { "center": { "longitude": -1.0769422, "latitude": 39.1398602 }, "radius": 20 } }], "VNFs": [{ "VNFid": "v1", "instances": 1, "requirements": { "cpu": 1, "ram": 100, "storage": 1 }, "failure_rate": 0, "processing_latency": 0, "CP": [{ "cpId": "CP1" }, { "cpId": "CP2", "VNFLink": { "id": "v1-v2-1", "latency": 12, "required_capacity": 30, "traversal_probability": 0 } }, { "cpId": "CP4", "VNFLink": { "id": "v1-v2-0", "latency": 12, "required_capacity": 10, "traversal_probability": 0 } }, { "cpId": "CP3", "VNFLink": { "id": "v1-v4-0", "latency": 28, "required_capacity": 5, "traversal_probability": 0 } }] }, { "VNFid": "v2", "instances": 1, "requirements": { "cpu": 1, "ram": 100, "storage": 1 }, "failure_rate": 0, "processing_latency": 0, "CP": [{ "cpId": "CP5", "VNFLink": { "id": "v1-v2-0", "latency": 12, "required_capacity": 10, "traversal_probability": 0 } }, { "cpId": "CP6", "VNFLink": { "id": "v1-v2-1", "latency": 12, "required_capacity": 30, "traversal_probability": 0 } }, { "cpId": "CP7", "VNFLink": { "id": "v2-v3-0", "latency": 12, "required_capacity": 20, "traversal_probability": 0 } }] }, { "VNFid": "v3", "instances": 1, "requirements": { "cpu": 1, "ram": 1, "storage": 1 }, "failure_rate": 0, "processing_latency": 0, "CP": [{ "cpId": "CP8", "VNFLink": { "id": "v2-v3-0", "latency": 12, "required_capacity": 20, "traversal_probability": 0 } }, { "cpId": "CP9", "VNFLink": { "id": "v3-v4-0", "latency": 40, "required_capacity": 10, "traversal_probability": 0 } }] }, { "VNFid": "v4", "instances": 1, "requirements": { "cpu": 1, "ram": 1, "storage": 1, "mec": True }, "failure_rate": 0, "processing_latency": 0, "CP": [{ "cpId": "CP10", "VNFLink": { "id": "v3-v4-0", "latency": 40, "required_capacity": 10, "traversal_probability": 0 } }, { "cpId": "CP11", "VNFLink": { "id": "v1-v4-0", "latency": 28, "required_capacity": 5, "traversal_probability": 0 } }] }], "VNFLinks": [{ "id": "v1-v2-0", "latency": 12, "required_capacity": 10, "traversal_probability": 0 }, { "id": "v1-v2-1", "latency": 12, "required_capacity": 30, "traversal_probability": 0 }, { "id": "v2-v3-0", "latency": 12, "required_capacity": 20, "traversal_probability": 0 }, { "id": "v3-v4-0", "latency": 40, "required_capacity": 10, "traversal_probability": 0 }, { "id": "v1-v4-0", "latency": 28, "required_capacity": 5, "traversal_probability": 0 }], "max_latency": 100, "target_availability": 0, "max_cost": 0 }, "callback": "string" } self.conn.request("POST", path, dumps(data), self.headers) self.conn.sock.settimeout(self.timeout) rsp = self.conn.getresponse() data = rsp.read() self.assertEqual(rsp.status, 201)
#print b64data # if b64data[len(b64data)-2]=='=': # b64data=b64data[:len(b64data)-2]+'%3d%3d' # elif b64data[len(b64data)-1]=='=': # b64data=b64data[:len(b64data)-1]+'%3d' # params='data='+b64data+'&vcs=8.1.1' params = s + '&browser=1&vcs=6.1.1' headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'Dalvik/1.6.0 (Linux; U; Android 4.1.2; GT-N7100 Build/JZO54K)', 'Host': Host, 'Connection': 'keep-alive', 'Accept-Encoding': 'gzip', 'Content-Length': len(params), } #print params conn = HTTPConnection(Host) conn.request('POST', '/ajax/sendChat.php', params, headers) #conn.close() #time.sleep(3) #print i result = conn.getresponse() data = result.read() print data #f=f+1 #print f #time.sleep(10) #res=json.loads(data) #print res['city_unit_count']['c285']
def post(data): c = HTTPConnection(ip) c.request('POST','/servlet/MIMEReceiveServlet',data,{'content-type':'text/xml'}) return BeautifulSoup(c.getresponse().read())
raise err sleep(0.1) return None def kill_server(port, port2server, pids): try: kill(pids[port2server[port]], SIGTERM) except Exception, err: print err try_until = time() + 30 while True: try: conn = HTTPConnection('127.0.0.1', port) conn.request('GET', '/') conn.getresponse() except Exception, err: break if time() > try_until: raise Exception('Still answering on port %s after 30 seconds' % port) sleep(0.1) def kill_servers(port2server, pids): for port in port2server: kill_server(port, port2server, pids) def kill_nonprimary_server(primary_nodes, port2server, pids): primary_ports = [n['port'] for n in primary_nodes]
class DynectRest(object): """ A class for interacting with the Dynect Managed DNS REST API. @ivar host: The host to connect to (defaults to api.dynect.net) @type host: C{str} @ivar port: The port to connect to (defaults to 443) @type port: C{int} @ivar ssl: A boolean indicating whether or not to use SSL encryption (defaults to True) @type ssl: C{bool} @ivar poll_incomplete: A boolean indicating whether we should continue to poll for a result if a job comes back as incomplete (defaults to True) @type poll_incomplete: C{bool} @ivar api_version: The version of the API to request (defaults to "current") @type api_version: C{str} """ def __init__(self, host='api.dynect.net', port=443, ssl=True, api_version="current"): """ Basic initializer method @param host: The host to connect to @type host: C{str} @param port: The port to connect to @type port: C{int} @param ssl: A boolean indicating whether or not to use SSL encryption @type ssl: C{bool} """ self.host = host self.port = port self.ssl = ssl # Continue polling for response if a job comes back as incomplete? self.poll_incomplete = True self.verbose = False self.api_version = api_version self.content_type = "application/json" self._token = None self._conn = None self._last_response = None self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT')) def _debug(self, msg): """ Debug output. """ if self.verbose: sys.stderr.write(msg) def connect(self): """ Establishes a connection to the REST API server as defined by the host, port and ssl instance variables """ if self._token: self._debug("Forcing logout from old session.\n") orig_value = self.poll_incomplete self.poll_incomplete = False self.execute('/REST/Session', 'DELETE') self.poll_incomplete = orig_value self._token = None self._conn = None if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % (self.host, self.port) self._debug(msg) self._conn = HTTPSConnection(self.host, self.port) else: msg = "Establishing unencrypted connection to %s:%s\n" % ( self.host, self.port) self._debug(msg) self._conn = HTTPConnection(self.host, self.port) def execute(self, uri, method, args=None): """ Execute a commands against the rest server @param uri: The uri of the resource to access. /REST/ will be prepended if it is not at the beginning of the uri. @type uri: C{str} @param method: One of 'DELETE', 'GET', 'POST', or 'PUT' @type method: C{str} @param args: Any arguments to be sent as a part of the request @type args: C{dict} """ if self._conn == None: self._debug("No established connection\n") self.connect() # Make sure the command is prefixed by '/REST/' if not uri.startswith('/'): uri = '/' + uri if not uri.startswith('/REST'): uri = '/REST' + uri # Make sure the method is valid if method.upper() not in self._valid_methods: msg = "%s is not a valid HTTP method. Please use one of %s" % ( method, ", ".join(self._valid_methods)) raise ValueError(msg) # Prepare arguments if args is None: args = {} args = self.format_arguments(args) self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args)) # Send the command and deal with results self.send_command(uri, method, args) # Deal with the results response = self._conn.getresponse() body = response.read() self._last_response = response if self.poll_incomplete: response, body = self.poll_response(response, body) self._last_response = response if sys.version_info[0] == 2: ret_val = json.loads(body) elif sys.version_info[0] == 3: ret_val = json.loads(body.decode('UTF-8')) self._meta_update(uri, method, ret_val) return ret_val def _meta_update(self, uri, method, results): """ Private method, not intended for use outside the class """ # If we had a successful log in, update the token if uri.startswith('/REST/Session') and method == 'POST': if results['status'] == 'success': self._token = results['data']['token'] # Otherwise, if it's a successful logout, blank the token if uri.startswith('/REST/Session') and method == 'DELETE': if results['status'] == 'success': self._token = None def poll_response(self, response, body): """ Looks at a response from a REST command, and while indicates that the job is incomplete, poll for response """ while response.status == 307: time.sleep(1) uri = response.getheader('Location') self._debug("Polling %s\n" % uri) self.send_command(uri, "GET", '') response = self._conn.getresponse() body = response.read() return response, body def send_command(self, uri, method, args): """ Responsible for packaging up the API request and sending it to the server over the established connection @param uri: The uri of the resource to interact with @type uri: C{str} @param method: The HTTP method to use @type method: C{str} @param args: Encoded arguments to send to the server @type args: C{str} """ if '%' not in uri: uri = pathname2url(uri) self._conn.putrequest(method, uri) # Build headers headers = { 'Content-Type': self.content_type, 'API-Version': self.api_version, } if self._token is not None: headers['Auth-Token'] = self._token for key, val in headers.items(): self._conn.putheader(key, val) # Now the arguments self._conn.putheader('Content-length', '%d' % len(args)) self._conn.endheaders() if sys.version_info[0] == 2: self._conn.send(args) elif sys.version_info[0] == 3: self._conn.send(bytes(args, 'UTF-8')) def format_arguments(self, args): """ Converts the argument dictionary to the format needed to transmit the REST request. @param args: Arguments to be passed to the REST call @type args: C{dict} @return: The encoded string to send in the REST request body @rtype: C{str} """ args = json.dumps(args) return args
use_s3 = settings.USE_S3 if use_s3: s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) bucket = s3.get_bucket(settings.AWS_VIDEO_BUCKET_NAME) ec2_name = settings.INSTANCE_NAME tmp_folder = os.path.dirname(os.path.abspath(__file__)) if not use_s3: tmp_folder = os.path.join(tmp_folder, os.path.pardir, os.path.pardir, 'media', 'tmp') conn = HTTPConnection(settings.DOMAIN_NAME) conn.request("GET", "/amazon/ec2-ready/%s/" % settings.INSTANCE_ID) conn.getresponse() def encode(ch, method, properties, body): """Fetches the temporary video file from s3 and encodes both web and mobile versions. Communicates with the screenbird server with the following - sets the `ec2_node` field of the associated `VideoStatus` for the video for identification of where to get encoding progress output of ffmpeg - sets the `web_available` field of the associated `VideoStatus` for the video after successfully encoding the web version - sets the `mobile_available` field of the associated `VideoStatus` for the video after successfully encoding the mobile version - sets the `video_duration` field of the associated `Video` object
def __call__ (self, host, req): conn = HTTPConnection (host) conn.request ("GET", req) r = conn.getresponse() return r.read()
class SingleDownload(): def __init__(self, downloader, url): self.downloader = downloader self.baseurl = url try: (self.scheme, self.netloc, path, pars, query, fragment) = urlparse(url) except: self.downloader.errorfunc('cannot parse http seed address: ' + url) return if self.scheme != 'http': self.downloader.errorfunc('http seed url not http: ' + url) return # Arno, 2010-03-08: Make proxy aware self.proxyhost = find_proxy(url) try: if self.proxyhost is None: self.connection = HTTPConnection(self.netloc) else: self.connection = HTTPConnection(self.proxyhost) except: self.downloader.errorfunc('cannot connect to http seed: ' + url) return self.seedurl = path if pars: self.seedurl += ';' + pars self.seedurl += '?' if query: self.seedurl += query + '&' self.seedurl += 'info_hash=' + urllib.quote(self.downloader.infohash) self.measure = Measure(downloader.max_rate_period) self.index = None self.url = '' self.requests = [] self.request_size = 0 self.endflag = False self.error = None self.retry_period = 30 self._retry_period = None self.errorcount = 0 self.goodseed = False self.active = False self.cancelled = False self.resched(randint(2, 10)) def resched(self, len=None): if len is None: len = self.retry_period if self.errorcount > 3: len = len * (self.errorcount - 2) self.downloader.rawserver.add_task(self.download, len) def _want(self, index): if self.endflag: return self.downloader.storage.do_I_have_requests(index) else: return self.downloader.storage.is_unstarted(index) def download(self): if DEBUG: print "http-sdownload: download()" self.cancelled = False if self.downloader.picker.am_I_complete(): self.downloader.downloads.remove(self) return self.index = self.downloader.picker.next(haveall, self._want, self) if self.index is None: self.resched(0.01) return if (self.index is None and not self.endflag and not self.downloader.peerdownloader.has_downloaders()): self.endflag = True self.index = self.downloader.picker.next(haveall, self._want, self) if self.index is None: self.endflag = True self.resched() else: self.url = (self.seedurl + '&piece=' + str(self.index)) self._get_requests() if self.request_size < self.downloader.storage._piecelen( self.index): self.url += '&ranges=' + self._request_ranges() rq = Thread(target=self._request) rq.setName("HoffmanHTTPDownloader" + rq.getName()) rq.setDaemon(True) rq.start() self.active = True def _request(self): import encodings.ascii import encodings.punycode import encodings.idna self.error = None self.received_data = None try: self.connection.request('GET', self.url, None, {'User-Agent': VERSION}) r = self.connection.getresponse() self.connection_status = r.status self.received_data = r.read() except Exception, e: print_exc() self.error = 'error accessing http seed: ' + str(e) try: self.connection.close() except: pass try: self.connection = HTTPConnection(self.netloc) except: self.connection = None # will cause an exception and retry next cycle self.downloader.rawserver.add_task(self.request_finished)
def encode(ch, method, properties, body): """Fetches the temporary video file from s3 and encodes both web and mobile versions. Communicates with the screenbird server with the following - sets the `ec2_node` field of the associated `VideoStatus` for the video for identification of where to get encoding progress output of ffmpeg - sets the `web_available` field of the associated `VideoStatus` for the video after successfully encoding the web version - sets the `mobile_available` field of the associated `VideoStatus` for the video after successfully encoding the mobile version - sets the `video_duration` field of the associated `Video` object """ param = eval(body) slug_tmp = param[0] host = param[1] mp4 = os.path.join(tmp_folder, "%s.mp4" % slug_tmp) slug = slug_tmp.replace("_tmp", "") text_file = os.path.join(tmp_folder, "%s.txt" % slug) output_mp4 = os.path.join(tmp_folder, "%s.mp4" % slug) output_mobile_mp4 = os.path.join(tmp_folder, "%s__mobile.mp4" % slug) if not use_s3: shutil.copyfile(output_mp4, mp4) if use_s3: key = bucket.get_key(slug_tmp) else: key = slug logger.info("Started encoding for %s" % slug_tmp) # Define these as variables to make it easier to change later on if need be set_encoding_url = "/set-encoding-ec2/%s/%s/" set_available_url = "/set-available/%s/%s/" time_diff = None if key: if use_s3: # Download source video to be encoded try: key.get_contents_to_filename(mp4) except: ch.basic_ack(delivery_tag=method.delivery_tag) # Notify requesting server that video is being processed try: conn = HTTPConnection(host) conn.request("GET", set_encoding_url % (slug, ec2_name)) response = conn.getresponse() except Exception, e: pass logger.info("Encoding for %s - web version" % slug_tmp) available = False # Check from requesting server if video is already encoded try: conn = HTTPConnection(host) conn.request("GET", "/is-available/%s/%s/" % (slug, 'web')) response = conn.getresponse() available = eval(response.read()) except: pass try: # Start encoding for video if not yet encoded if not available: start_time = datetime.datetime.now() encode_video({ 'mp4': str(mp4), 'output_mp4': str(output_mp4), 'text_file': str(text_file) }) end_time = datetime.datetime.now() time_diff = end_time - start_time logger.info("Web encoding time: %s" % time_diff) else: logger.info( "Encoded version for %s already available - web version" % slug_tmp) except Exception, e: logger.info("Encoding for %s failed - web version: %s" % (slug_tmp, e)) ch.basic_ack(delivery_tag=method.delivery_tag) ch.basic_publish(exchange='', routing_key=settings.QUEUE_NAME, body=body)
publish_interval = 1 pub = Publisher(8290) server = "rtgpsout.unavco.org:2101" headers = { 'Ntrip-Version': 'Ntrip/2.0', 'User-Agent': 'NTRIP ntrip_ros', 'Connection': 'close', 'Authorization': 'Basic ' + b64encode(user) } connection = HTTPConnection(server) connection.request('GET', '/SLAC_RTCM3', body=None, headers=headers) response = connection.getresponse() # Message format # https://forum.u-blox.com/index.php/16898/decoding-rtcm3-message def get_length(msg): assert (msg[0] == chr(0xd3)) return (ord(msg[1]) * 8 + ord(msg[2])) & 0x3ff buf = "" msgs = [] last_message = time.time() while 1:
def validateURL(url, firstOccurrenceOnly=1, wantRawData=0, groupEvents=0): """validate RSS from URL, returns events list, or (events, rawdata) tuple""" loggedEvents = [] request = urllib2.Request(url) request.add_header("Accept-encoding", "gzip, deflate") request.add_header("User-Agent", "FeedValidator/1.3") usock = None ctx2 = ssl.create_default_context() ctx1 = ssl.create_default_context() ctx2.set_ciphers('ALL:@SECLEVEL=2') ctx1.set_ciphers('ALL:@SECLEVEL=1') try: try: try: usock = urllib2.urlopen(request, context=ctx2) except urllib2.URLError as x: import re if True or re.search("WRONG_SIGNATURE_TYPE", x.reason): loggedEvents.append( HttpsProtocolWarning( {'message': "Weak signature used by HTTPS server"})) usock = urllib2.urlopen(request, context=ctx1) else: raise x rawdata = usock.read(MAXDATALENGTH) if usock.read(1): raise ValidationFailure( logging.ValidatorLimit({ 'limit': 'feed length > ' + str(MAXDATALENGTH) + ' bytes' })) # check for temporary redirects if usock.geturl() != request.get_full_url(): from urlparse import urlsplit (scheme, netloc, path, query, fragment) = urlsplit(url) if scheme == 'http': from httplib import HTTPConnection requestUri = (path or '/') + (query and '?' + query) conn = HTTPConnection(netloc) conn.request("GET", requestUri) resp = conn.getresponse() if resp.status != 301: loggedEvents.append(TempRedirect({})) except BadStatusLine as status: raise ValidationFailure( logging.HttpError({'status': status.__class__})) except urllib2.HTTPError as status: rawdata = status.read() if len(rawdata) < 512 or 'content-encoding' in status.headers: loggedEvents.append(logging.HttpError({'status': status})) usock = status else: rawdata = re.sub('<!--.*?-->', '', rawdata) lastline = rawdata.strip().split('\n')[-1].strip() if sniffPossibleFeed(rawdata): loggedEvents.append(logging.HttpError({'status': status})) loggedEvents.append(logging.HttpErrorWithPossibleFeed({})) usock = status else: raise ValidationFailure( logging.HttpError({'status': status})) except urllib2.URLError as x: raise ValidationFailure(logging.HttpError({'status': x.reason})) except Timeout as x: raise ValidationFailure( logging.IOError({ "message": 'Server timed out', "exception": x })) except Exception as x: raise ValidationFailure( logging.IOError({ "message": x.__class__.__name__, "exception": x })) if usock.headers.get('content-encoding', None) == None: loggedEvents.append(Uncompressed({})) if usock.headers.get('content-encoding', None) == 'gzip': import gzip, StringIO try: rawdata = gzip.GzipFile( fileobj=StringIO.StringIO(rawdata)).read() except: import sys exctype, value = sys.exc_info()[:2] event = logging.IOError({ "message": 'Server response declares Content-Encoding: gzip', "exception": value }) raise ValidationFailure(event) if usock.headers.get('content-encoding', None) == 'deflate': import zlib try: rawdata = zlib.decompress(rawdata, -zlib.MAX_WBITS) except: import sys exctype, value = sys.exc_info()[:2] event = logging.IOError({ "message": 'Server response declares Content-Encoding: deflate', "exception": value }) raise ValidationFailure(event) if usock.headers.get('content-type', None) == 'application/vnd.google-earth.kmz': import tempfile, zipfile, os try: (fd, tempname) = tempfile.mkstemp() os.write(fd, rawdata) os.close(fd) zfd = zipfile.ZipFile(tempname) namelist = zfd.namelist() for name in namelist: if name.endswith('.kml'): rawdata = zfd.read(name) zfd.close() os.unlink(tempname) except: import sys value = sys.exc_info()[:1] event = logging.IOError({ "message": 'Problem decoding KMZ', "exception": value }) raise ValidationFailure(event) mediaType = None charset = None # Is the Content-Type correct? contentType = usock.headers.get('content-type', None) if contentType: (mediaType, charset) = mediaTypes.checkValid(contentType, loggedEvents) # Check for malformed HTTP headers for (h, v) in usock.headers.items(): if (h.find(' ') >= 0): loggedEvents.append(HttpProtocolError({'header': h})) selfURIs = [request.get_full_url()] baseURI = usock.geturl() if not baseURI in selfURIs: selfURIs.append(baseURI) # Get baseURI from content-location and/or redirect information if usock.headers.get('content-location', None): from urlparse import urljoin baseURI = urljoin(baseURI, usock.headers.get('content-location', "")) elif usock.headers.get('location', None): from urlparse import urljoin baseURI = urljoin(baseURI, usock.headers.get('location', "")) if not baseURI in selfURIs: selfURIs.append(baseURI) usock.close() usock = None mediaTypes.contentSniffing(mediaType, rawdata, loggedEvents) encoding, rawdata = xmlEncoding.decode(mediaType, charset, rawdata, loggedEvents, fallback='utf-8') if rawdata is None: return {'loggedEvents': loggedEvents} rawdata = rawdata.replace('\r\n', '\n').replace('\r', '\n') # normalize EOL validator = _validate(rawdata, firstOccurrenceOnly, loggedEvents, baseURI, encoding, selfURIs, mediaType=mediaType, groupEvents=groupEvents) # Warn about mismatches between media type and feed version if mediaType and validator.feedType: mediaTypes.checkAgainstFeedType(mediaType, validator.feedType, validator.loggedEvents) params = { "feedType": validator.feedType, "loggedEvents": validator.loggedEvents } if wantRawData: params['rawdata'] = rawdata return params finally: try: if usock: usock.close() except: pass