def open_local_file(self, req): host = req.get_host() file = req.get_selector() localfile = urllib2.url2pathname(file) stats = os.stat(localfile) size = stats.st_size modified = formatdate(stats.st_mtime, usegmt=True) mtype = mimetypes.guess_type(file)[0] headers = mimetools.Message(cStringIO.StringIO( 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % (mtype or 'text/plain', size, modified))) if host: host, port = urllib.splitport(host) if not host or \ (not port and socket.gethostbyname(host) in self.get_names()): try: file_list = dircache.listdir(localfile) s = cStringIO.StringIO() s.write('<html><head><base href="%s"/></head><body>' % ('file:' + file)) s.write('<p>Directory Content:</p>') for f in file_list: s.write('<p><a href="%s">%s</a></p>\n' % (urllib.quote(f), f)) s.write('</body></html>') s.seek(0) headers = mimetools.Message(cStringIO.StringIO( 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % ('text/html', size, modified))) return urllib2.addinfourl(s, headers, 'file:' + file) except OSError: return urllib2.addinfourl(open(localfile, 'rb'), headers, 'file:'+file) raise urllib2.URLError('file not on local host')
def open_local_file(self, req): import email.Utils host = req.get_host() file = req.get_selector() localfile = url2pathname(file) stats = os.stat(localfile) size = stats.st_size modified = email.Utils.formatdate(stats.st_mtime, usegmt=True) mtype = mimetypes.guess_type(file)[0] headers = mimetools.Message(StringIO( 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % (mtype or 'text/plain', size, modified))) if host: host, port = splitport(host) if not host or \ (not port and socket.gethostbyname(host) in self.get_names()): try: file_list = dircache.listdir(localfile) s = StringIO() s.write('<html><head><base href="%s"/></head><body>' % ('file:' + file)) s.write('<p>Directory Content:</p>') for f in file_list: s.write('<p><a href="%s">%s</a></p>\n' % (urllib.quote(f), f)) s.write('</body></html>') s.seek(0) headers = mimetools.Message(StringIO( 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' % ('text/html', size, modified))) return addinfourl(s, headers, 'file:' + file) except OSError: return addinfourl(open(localfile, 'rb'), headers, 'file:'+file) raise URLError('file not on local host')
def check_zip(self, resp): """ @Def: check_zip @Description: check web page if it's ziped @Input: resp #response of web @Output: data #web page data @Date: 2015-8-10 """ if resp.headers.get("content-encoding") == "gzip": lib.log_print.log_print_debug("Zip page.") data = resp.read() gz = gzip.GzipFile(fileobj=StringIO.StringIO(data)) new_resp = urllib2.addinfourl(gz, resp.headers, resp.url, resp.code) new_resp.msg = resp.msg return new_resp elif resp.headers.get("content-encoding") == "deflate": lib.log_print.log_print_debug(" Deflate page. url: %s" % (resp.geturl())) data = resp.read() gz = StringIO.StringIO(self._deflate(data)) new_resp = urllib2.addinfourl(gz, resp.headers, resp.url, resp.code) new_resp.msg = resp.msg return new_resp else: lib.log_print.log_print_debug( " Unzip page. Encoding type: %s, url: %s" % (str(resp.headers.get("content-encoding")), resp.geturl()) ) return resp
def GetLastUpdateTime(): last_update = None url = 'http://hq.sinajs.cn/rn=1389255083862&list=gb_dji' data = '' user_agent = 'Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20100101 Firefox/12.0' headers = { 'User-Agent' : user_agent, 'Host': 'hq.sinajs.cn', 'Accept': '*/*', 'Accept-Language': 'en-us,en;q=0.5', 'Accept-Encoding': 'gzip, deflate', #'Connection': 'keep-alive', 'Referer': 'http://finance.sina.com.cn/stock/usstock/sector.shtml', } req = urllib2.Request(url, data, headers) resp = urllib2.urlopen(req) old_resp = resp if resp.headers.get("content-encoding") == "gzip": gz = GzipFile( fileobj=StringIO(resp.read()), mode="r" ) #resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url) resp.msg = old_resp.msg #json_html = gz.read() #print 'xxx' # deflate if resp.headers.get("content-encoding") == "deflate": gz = StringIO( deflate(resp.read()) ) #resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) # 'class to add info() and resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url) # 'class to add info() and resp.msg = old_resp.msg #json_html = gz.read() #print 'YY' js_script = resp.read() #print js_script last_update_time = js_script.split(',')[25] #print last_update_time #Jan 08 04:13PM EST ridx = last_update_time.rfind('EST') if ridx == -1: #try: raise ValueError('Timezone EST value not set') #except ValueError, e: # print e else: last_update_time = last_update_time[:ridx].strip() print last_update_time last_update = datetime.datetime.strptime(last_update_time, "%b %d %I:%M%p") last_update = last_update.replace(datetime.date.today().year) print last_update return last_update
def GetLastUpdateTime(): last_update = None url = 'http://hq.sinajs.cn/rn=1389255083862&list=gb_dji' data = '' user_agent = 'Mozilla/5.0 (Windows NT 5.1; rv:12.0) Gecko/20100101 Firefox/12.0' headers = { 'User-Agent': user_agent, 'Host': 'hq.sinajs.cn', 'Accept': '*/*', 'Accept-Language': 'en-us,en;q=0.5', 'Accept-Encoding': 'gzip, deflate', #'Connection': 'keep-alive', 'Referer': 'http://finance.sina.com.cn/stock/usstock/sector.shtml', } req = urllib2.Request(url, data, headers) resp = urllib2.urlopen(req) old_resp = resp if resp.headers.get("content-encoding") == "gzip": gz = GzipFile(fileobj=StringIO(resp.read()), mode="r") #resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url) resp.msg = old_resp.msg #json_html = gz.read() #print 'xxx' # deflate if resp.headers.get("content-encoding") == "deflate": gz = StringIO(deflate(resp.read())) #resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) # 'class to add info() and resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url) # 'class to add info() and resp.msg = old_resp.msg #json_html = gz.read() #print 'YY' js_script = resp.read() #print js_script last_update_time = js_script.split(',')[25] #print last_update_time #Jan 08 04:13PM EST ridx = last_update_time.rfind('EST') if ridx == -1: #try: raise ValueError('Timezone EST value not set') #except ValueError, e: # print e else: last_update_time = last_update_time[:ridx].strip() print last_update_time last_update = datetime.datetime.strptime(last_update_time, "%b %d %I:%M%p") last_update = last_update.replace(datetime.date.today().year) print last_update return last_update
def http_response(self, request, response): old_response = response if response.headers.get('Content-Encoding') == 'gzip': gz = gzip.GzipFile(fileobj=StringIO.StringIO(response.read() ), mode='r') response = urllib2.addinfourl(gz, old_response.headers, old_response.url, old_response.code) response.msg = old_response.msg if response.headers.get('Content-Encoding') == 'deflate': gz = StringIO.StringIO(zlib.decompressobj(-zlib.MAX_WBITS).decompress(response.read() ) ) response = urllib2.addinfourl(gz, old_response.headers, old_response.url, old_response.code) response.msg = old_response.msg return response
def http_response(self, request, response): old_response = response if response.headers.get("content-encoding") == "gzip": gzipFile = gzip.GzipFile(fileobj=StringIO.StringIO(response.read()),mode="r") response = urllib2.addinfourl(gzipFile, old_response.headers, old_response.url, old_response.code) response.msg = old_response.msg if response.headers.get("content-encoding") == "deflate": gzipFile = StringIO.StringIO(deflate(response.read()) ) response = urllib2.addinfourl(gzipFile, old_response.headers, old_response.url, old_response.code) response.msg = old_response.msg return response
def http_response(self, req, resp): old_resp = resp if resp.headers.get("content-encoding") == "gzip": gz = gzip.GzipFile( fileobj=BytesIO(resp.read()), mode="r" ) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg # deflate if resp.headers.get("content-encoding") == "deflate": gz = BytesIO( self.deflate(resp.read()) ) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def http_response(self, req, resp): old_resp = resp # gzip if resp.headers.get('content-encoding') == 'gzip': gz = GzipFile(fileobj=BytesIO(resp.read()), mode='r') resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg # deflate if resp.headers.get('content-encoding') == 'deflate': gz = BytesIO( deflate(resp.read()) ) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) # 'class to add info() and resp.msg = old_resp.msg return resp
def http_response(self, req, resp): old_resp = resp if resp.headers.get("content-encoding") == "gzip": data = gzip.GzipFile(fileobj=StringIO(resp.read()), mode="r") resp = urllib2.addinfourl(data, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg if resp.headers.get("content-encoding") == "deflate": data = zlib.decompress(resp.read(), -zlib.MAX_WBITS) resp = urllib2.addinfourl(data, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def mock_response(self, req): if req.get_full_url() == "http://localhost/test": resp = urllib2.addinfourl(StringIO("mock file"), "mock message", req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp if req.get_full_url() == "http://localhost/typo": resp = urllib2.addinfourl(StringIO("mock file"), "mock message", req.get_full_url()) resp.code = 404 resp.msg = "Not Found" return resp else: raise SystemExit, 2
def http_response(self, req, resp): old_resp = resp if resp.headers.get('content-encoding') == 'gzip': gz = GzipFile(fileobj=StringIO(resp.read()), mode='r') resp = addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg if resp.headers.get('content-encoding') == 'deflate': gz = StringIO(deflate(resp.read())) resp = addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def http_response(self, req, resp): old_resp = resp # gzip if resp.headers.get("content-encoding") == "gzip": gz = GzipFile(fileobj=StringIO(resp.read()), mode="r") resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg # deflate if resp.headers.get("content-encoding") == "deflate": gz = StringIO(deflate(resp.read())) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) # 'class to add info() and resp.msg = old_resp.msg return resp
def http_open(self, req): url = req.get_full_url() try: content = self.urls[url] except KeyError: resp = urllib2.addinfourl(StringIO(""), None, url) resp.code = 404 resp.msg = "OK" else: resp = urllib2.addinfourl(StringIO(content), None, url) resp.code = 200 resp.msg = "OK" return resp
def http_response(self, req, resp): old_resp = resp # gzip if resp.headers.get("content-encoding") == "gzip": gz = GzipFile(fileobj=StringIO(resp.read()), mode="r") resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg # deflate if resp.headers.get("content-encoding") == "deflate": gz = StringIO(deflate(resp.read())) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def to_addinfourl(response): """Convert a `django.http.HttpResponse` to a `urllib2.addinfourl`.""" headers_raw = response.serialize_headers() headers = httplib.HTTPMessage(io.BytesIO(headers_raw)) return urllib2.addinfourl( fp=io.BytesIO(response.content), headers=headers, url=None, code=response.status_code)
def extract(resp): data = resp.read() # print type(data) # print data try: # print len(data) if resp.headers.get('content-encoding') == 'gzip': data = gzip(data) # print data if resp.headers.get('content-encoding') == 'deflate': data = deflate(data) # print data # print len(data) # print resp.headers # print resp.url # print resp.code # resp = urllib2.addinfourl(fp=data, headers=resp.headers, url=resp.url, code=resp.code) # print type(data) # print resp.headers # print len(resp.read()) # print dir(resp) except Exception as err: print err resp = urllib2.addinfourl(data, headers=resp.headers, url=resp.url, code=resp.code) # print resp.read() return resp
def test_urlopen_success(self): response = urllib2.addinfourl(self.fp, self.headers, self.url) response.code = 200 response.msg = "OK" with mock.patch('urllib2.urlopen') as mock_urlopen: mock_urlopen.return_value = response urlopen(self.url)
def http_open(self, req): if self.resp_func: return resp_func(req) if self.mock_url and self.mock_url != req.get_full_url(): resp = urllib2.addinfourl(StringIO('Not Found'), 'Not Found', req.get_full_url()) resp.code = 404 resp.msg = 'Not Found' return resp resp = urllib2.addinfourl(StringIO(self.mock_file), self.mock_msg, req.get_full_url()) resp.code = self.mock_code resp.msg = self.mock_resp_msg return resp
def http_open(self, req): requests.append(dict( url=req.get_full_url(), contenttype=req.headers['Content-type'], auth=base64.decodestring(req.headers['Authorization'].replace('Basic ', '')), data=json.loads(req.data), )) # Sanitise response global nextResponse if 'id' not in nextResponse: nextResponse['id'] = requests[-1]['data']['id'] if 'error' not in nextResponse: nextResponse['error'] = None if 'result' not in nextResponse: nextResponse['result'] = 1234 resp = urllib2.addinfourl( StringIO(nextResponse['_data'] if '_data' in nextResponse else json.dumps(nextResponse)), "Message of some form", req.get_full_url(), ) resp.code = nextResponse['_code'] if '_code' in nextResponse else 200 resp.msg = nextResponse['_msg'] if '_msg' in nextResponse else "OK" nextResponse = dict() return resp
def mock_response(req, resp_obj, resp_code): """ Mock response for MyHTTPSHandler """ resp = urllib2.addinfourl(StringIO(resp_obj), "This is a mocked URI!", req.get_full_url()) resp.code = resp_code resp.msg = "OK" return resp
def http_response(self, req, resp): # code for after-fetch, to know whether to save to hard-drive (if stiking to http headers' will) if resp.code == 304: return resp if ('cache-control' in resp.headers or 'pragma' in resp.headers) and self.force_min is None: cache_control = parse_http_list(resp.headers.get('cache-control', ())) cache_control += parse_http_list(resp.headers.get('pragma', ())) cc_list = [x for x in cache_control if '=' not in x] if 'no-cache' in cc_list or 'no-store' in cc_list or ('private' in cc_list and not self.private): # kindly follow web servers indications return resp if resp.headers.get('Morss') == 'from_cache': # it comes from cache, so no need to save it again return resp # save to disk data = resp.read() self._save(req.get_full_url(), resp.code, resp.msg, resp.headers, data, time.time()) fp = BytesIO(data) old_resp = resp resp = addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def response(self, req): data = self.content(req) url = req.get_full_url() resp = urllib2.addinfourl(StringIO(data), data, url) resp.code = self.code resp.msg = self.msg return resp
def http_open(self, req): if req.get_full_url().startswith("https://api.github.com/users/tester/subscriptions"): response_file = "test_data/good_subscription_response.json" resp = urllib2.addinfourl(open(response_file, "r"), "mock", req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def http_response(self, _, resp): if resp.headers.get("content-encoding") == "gzip": gz = GzipFile(fileobj=StringIO(resp.read()), mode="r") old_resp = resp resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def https_open(self, req): if '/tree/' in req.get_full_url(): resp = urllib2.addinfourl(StringIO(TEST_GET_JOBS_RETURN), "", req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def sftp_open(self, req): url = req.get_full_url() p = urlparse.urlparse(url) try: transport = paramiko.Transport((p.hostname, (p.port or 22))) transport.connect(username=p.username, password=p.password) sftp = paramiko.SFTPClient.from_transport(transport) filename = os.path.basename(p.path) path_to_file = os.path.dirname(p.path) sftp.chdir(path_to_file) size = sftp.stat(filename).st_size content_type = mimetypes.guess_type( filename)[0] or 'application/octet-stream' headers = {'Content-type': content_type, 'Content-length': size} fp = sftp.open(filename) return urllib2.addinfourl(fp, headers, url) except paramiko.ssh_exception.BadAuthenticationType as e: # Bad Login Credentials raise urllib2.HTTPError(url=url, code=403, msg=e, hdrs=None, fp=None) except IOError as e: # File not found raise urllib2.HTTPError(url=url, code=404, msg=e, hdrs=None, fp=None) except Exception as e: raise urllib2.HTTPError(url=url, code=400, msg=e, hdrs=None, fp=None)
def test_get_swarm_results_success(self): self.mox.StubOutWithMock(swarm_results.urllib2, 'urlopen') shard_output = json.dumps( {'machine_id': 'host', 'machine_tag': 'localhost', 'exit_codes': '0, 0', 'output': SWARM_OUTPUT_WITHOUT_FAILURE } ) url_response = urllib2.addinfourl(StringIO.StringIO(shard_output), "mock message", 'host') url_response.code = 200 url_response.msg = "OK" swarm_results.urllib2.urlopen('http://host:9001/get_result?r=key1' ).AndReturn(url_response) swarm_results.urllib2.urlopen('http://host:9001/cleanup_results', data=urllib.urlencode({'r': 'key1'}) ).AndReturn(StringIO.StringIO('')) self.mox.ReplayAll() swarm_results.GetSwarmResults('http://host:9001', ['key1']) self.checkstdout(BUILDBOT_OUTPUT) self.mox.VerifyAll()
def mock_response(req): if req.get_full_url() == ip_address_uri: mock_file = open('test_pytomo/OdF-oiaICZI.flv') resp = urllib2.addinfourl(mock_file, info, req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def mock_response(req): url= req.get_full_url().split('/')[-1] if url in URLS.keys(): resp = urllib2.addinfourl(StringIO.StringIO(URLS[url]), "mock message", req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def mock_response(req): url = req.get_full_url() if url == 'http://geocoder.demozoo.org/?q=Oxford': body = """[ {"name": "Oxford, Oxfordshire, England, United Kingdom", "id": 2640729}, {"name": "Oxford, Butler County, Ohio, United States", "id": 4520760}, {"name": "Oxford, Calhoun County, Alabama, United States", "id": 4081914} ]""" elif url == 'http://geocoder.demozoo.org/2640729/': body = """{ "full_name": "Oxford, Oxfordshire, England, United Kingdom", "latitude": 51.75222, "longitude": -1.25596, "country_name": "United Kingdom", "name": "Oxford", "id": 2640729, "country_code": "GB" }""" elif url == 'http://geocoder.demozoo.org/?q=Adlington%2C+Lancashire%2C+England%2C+United+Kingdom': raise Exception("Looking up Adlington is not allowed! :-)") elif url == 'http://geocoder.demozoo.org/?q=Royston+Vasey': body = "[]" else: raise Exception("No response defined for %s" % req.get_full_url()) resp = urllib2.addinfourl(StringIO(body), None, req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def http_response(self, req, resp): contenttype = resp.info().get('Content-Type', '').split(';')[0] if 200 <= resp.code < 300 and len( self.follow) and contenttype in MIMETYPE[ 'html'] and contenttype not in self.follow: # opps, not what we were looking for, let's see if the html page suggests an alternative page of the right types data = resp.read() links = lxml.html.fromstring( data[:10000]).findall('.//link[@rel="alternate"]') for link in links: if link.get('type', '') in self.follow: resp.code = 302 resp.msg = 'Moved Temporarily' resp.headers['location'] = link.get('href') break fp = BytesIO(data) old_resp = resp resp = addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def http_open(self, req): req_url = req.get_full_url() file_path = req_url.replace(base_http_url, base_file_path) resp = urllib2.addinfourl(open(file_path), "redirected to file", req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def mock_response(req): if req.get_full_url() == "http://www.npr.org/rss/rss.php?id=1001": resp = urllib2.addinfourl(StringIO("mock response:"), "mock message", req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def test_get_swarm_results_success(self): self.mox.StubOutWithMock(swarm_results.urllib2, 'urlopen') shard_output = json.dumps( {'hostname': 'host', 'exit_codes': '0, 0', 'output': SWARM_OUTPUT_WITHOUT_FAILURE } ) url_response = urllib2.addinfourl(StringIO.StringIO(shard_output), "mock message", 'host') url_response.code = 200 url_response.msg = "OK" swarm_results.urllib2.urlopen('http://host:9001/get_result?r=key1' ).AndReturn(url_response) swarm_results.urllib2.urlopen('http://host:9001/cleanup_results', mox.IgnoreArg()) self.mox.ReplayAll() swarm_results.GetSwarmResults('http://host:9001', ['key1']) self.checkstdout(BUILDBOT_OUTPUT) self.mox.VerifyAll()
def http_response(self, req, resp): # code for after-fetch, to know whether to save to hard-drive (if stiking to http headers' will) if resp.code == 304: return resp if ('cache-control' in resp.headers or 'pragma' in resp.headers) and self.force_min is None: cache_control = parse_http_list( resp.headers.get('cache-control', ())) cache_control += parse_http_list(resp.headers.get('pragma', ())) cc_list = [x for x in cache_control if '=' not in x] if 'no-cache' in cc_list or 'no-store' in cc_list or ( 'private' in cc_list and not self.private): # kindly follow web servers indications return resp if resp.headers.get('Morss') == 'from_cache': # it comes from cache, so no need to save it again return resp # save to disk data = resp.read() self._save(req.get_full_url(), resp.code, resp.msg, resp.headers, data, time.time()) fp = BytesIO(data) old_resp = resp resp = addinfourl(fp, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def fake_response(self, req): grader_endpoint = getattr(settings, 'GRADER_ENDPOINT', 'localhost') if req.get_full_url() == grader_endpoint: resp = urllib2.addinfourl(StringIO(self.answer), "", req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def test_ignore_non_index_etag(self): """ Etags are ignored for non-index requests """ req = urllib2.Request('http://foo.com/') res = urllib2.addinfourl(StringIO('foo'), {'Etag': 'whatevs'}, 'http://foo.com/') self.cache_handler.http_response(req, res) self.assertFalse(os.path.exists(self.metadata_path)) self.assertFalse(os.path.exists(self.index_path))
def http_response(self, req, resp): old_resp = resp if not req.has_header('Accept-encoding'): return resp if req.has_header('Range'): return resp if resp.headers.get("content-encoding") == "gzip": gz = _GzipFile(fileobj=StringIO(resp.read()), mode="r") resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg # deflate if resp.headers.get("content-encoding") == "deflate": gz = StringIO(self.deflate(resp.read())) resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) resp.msg = old_resp.msg return resp
def mock_response(req): if req.get_full_url() == 'http://sourcetest.com/rss.xml': resp = urllib2.addinfourl(StringIO.StringIO(rss_source_xml), 'mock message', req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def test_no_etag(self): """ No Etag header == no cache written """ req = urllib2.Request('http://foo.com/index.json') res = urllib2.addinfourl(StringIO('foo'), {}, 'http://foo.com/index.json') self.cache_handler.http_response(req, res) self.assertFalse(os.path.exists(self.metadata_path)) self.assertFalse(os.path.exists(self.index_path))
def testscheme_open(self, req): try: selector = req.get_selector() if selector == u'/ws_newcompass.asmx?WSDL': return urllib.addinfourl( pkg_resources.resource_stream(__name__, 'tests/testdata/wsdl.xml'), httplib.HTTPMessage(open('/dev/null')), req.get_full_url(), 200 ) elif selector == u'/ws_newcompass.asmx': soapResponse = urlparse.urlparse(req.get_header('Soapaction')).path.strip('"').split('/')[-1] + '.xml' return urllib.addinfourl( pkg_resources.resource_stream(__name__, 'tests/testdata/' + soapResponse), httplib.HTTPMessage(open('/dev/null')), req.get_full_url(), 200 ) elif selector == u'/biomuta.tsv': return urllib2.addinfourl( pkg_resources.resource_stream(__name__, 'tests/testdata/Biomuta.tsv'), httplib.HTTPMessage(open('/dev/null')), req.get_full_url(), 200 ) else: raise urllib2.URLError('Not found') except Exception: raise urllib2.URLError('Not found')
def mock_response(req): "Mock Response class" if req.get_full_url() == VIDEO_URL: mock_file = open(FLASH_FILE) resp = urllib2.addinfourl(mock_file, INFO, req.get_full_url()) resp.code = 200 resp.msg = "OK" return resp
def mock_responses(request, code, msg, cors): """Mock responses.""" response = urllib2.addinfourl(StringIO('mock file'), {'Access-Control-Allow-Origin': cors}, request.get_full_url()) response.code = code response.msg = msg return response
def generate_url_response(index, shard_output, exit_codes): url_response = urllib2.addinfourl( StringIO.StringIO(json.dumps(gen_data(index, shard_output, exit_codes))), 'mock message', 'host') url_response.code = 200 url_response.msg = 'OK' return url_response
def http_response(self, req, resp): """Handle encodings in the order that they are encountered.""" encodings = [] headers = resp.headers encoding_header = None for header in headers: if header.lower() == "content-encoding": encoding_header = header for encoding in headers[header].split(","): encoding = encoding.strip() if encoding: encodings.append(encoding) break if not encodings: return resp # encoding_header can't be None here as the above return on an empty list # of encodings would have prevented this line from being reached. del headers[encoding_header] fp = resp while encodings and encodings[-1].lower() == "gzip": fp = cStringIO.StringIO(fp.read()) fp = gzip.GzipFile(fileobj=fp, mode="r") encodings.pop() if encodings: # Some unhandled encodings remain, leave them for other handlers. # There may be further encodings that we can handle nested within the # unhandled encoding. # TODO(user): The only resolution is to support more encodings. headers[encoding_header] = ", ".join(encodings) logger.warning("Unrecognized Content-Encoding: %s", encodings[-1]) msg = resp.msg if sys.version_info >= (2, 6): resp = urllib2.addinfourl(fp, headers, resp.url, resp.code) else: response_code = resp.code resp = urllib2.addinfourl(fp, headers, resp.url) resp.code = response_code resp.msg = msg return resp
def http_error_302(self, req, fp, code, msg, headers): infourl = urllib2.addinfourl(fp, headers, req.get_full_url()) infourl.status = code infourl.code = code if headers['location']: return headers['location'] else: return infourl
def mock_response(self, req): if req.get_full_url() in self.res_dict.keys(): metadata = self.res_dict[req.get_full_url()] data = metadata['data'] headers = metadata['headers'] res_code = metadata['res_code'] res_msg = metadata['res_msg'] resp = urllib2.addinfourl(StringIO(data), headers, req.get_full_url()) resp.code = res_code resp.msg = res_msg return resp else: resp = urllib2.addinfourl(StringIO("The url was not found in my dictionary!"), "mock message", req.get_full_url()) resp.code = 404 resp.msg = "Error" return resp
def http_response(self, req, resp): if resp.headers.get('content-encoding') == 'gzip': gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r') old = resp resp = urllib2.addinfourl(gz, old.headers, old.url) resp.msg = old.msg resp.code = old.code # support for Python2.4/2.5 return resp
def to_addinfourl(response): """Convert a `django.http.HttpResponse` to a `urllib2.addinfourl`.""" headers_raw = response.serialize_headers() headers = httplib.HTTPMessage(io.BytesIO(headers_raw)) return urllib2.addinfourl(fp=io.BytesIO(response.content), headers=headers, url=None, code=response.status_code)