def test_request_redirect_follow_query_string(self):
redirect_url = 'http://urlb.com/?aa=a'
r = PageRedirect('redirect')
r.location = redirect_url
ru, iu = self.mock_all_identify(ru_side=r)
identify_line(self.lines[0])
args, kwargs = iu.call_args
self.assertEquals(args[0], 'http://urlb.com/')
def test_request_redirect_follow_query_string(self):
redirect_url = 'http://urlb.com/?aa=a'
r = PageRedirect('redirect')
r.location = redirect_url
ru, iu = self.mock_all_identify(ru_side=r)
identify_line(self.lines[0])
args, kwargs = iu.call_args
self.assertEquals(args[0], 'http://urlb.com/')
def test_request_redirect_follow(self):
redirect_url = 'http://urlb.com/'
r = PageRedirect('redirect')
r.location = redirect_url
ru, iu = self.mock_all_identify(ru_side=r)
identify_line(self.lines[0])
self.assertEquals(iu.call_count, 1)
args, kwargs = iu.call_args
self.assertEquals(args[0], redirect_url)
def test_request_redirect_follow(self):
redirect_url = 'http://urlb.com/'
r = PageRedirect('redirect')
r.location = redirect_url
ru, iu = self.mock_all_identify(ru_side=r)
identify_line(self.lines[0])
self.assertEquals(iu.call_count, 1)
args, kwargs = iu.call_args
self.assertEquals(args[0], redirect_url)
def gotHeaders(self, headers):
if self.code == http.OK:
if "content-length" in headers:
self.total_length = int(headers["content-length"][0])
else:
self.total_length = 0
if self.allow_compression and "content-encoding" in headers and \
headers["content-encoding"][0] in ("gzip", "x-gzip", "deflate"):
# Adding 32 to the wbits enables gzip & zlib decoding (with automatic header detection)
# Adding 16 just enables gzip decoding (no zlib)
self.decoder = zlib.decompressobj(zlib.MAX_WBITS + 32)
if "content-disposition" in headers and not self.force_filename:
new_file_name = str(headers["content-disposition"][0]).split(";")[1].split("=")[1]
new_file_name = sanitise_filename(new_file_name)
new_file_name = os.path.join(os.path.split(self.fileName)[0], new_file_name)
count = 1
fileroot = os.path.splitext(new_file_name)[0]
fileext = os.path.splitext(new_file_name)[1]
while os.path.isfile(new_file_name):
# Increment filename if already exists
new_file_name = "%s-%s%s" % (fileroot, count, fileext)
count += 1
self.fileName = new_file_name
self.value = new_file_name
elif self.code in (http.MOVED_PERMANENTLY, http.FOUND, http.SEE_OTHER, http.TEMPORARY_REDIRECT):
location = headers["location"][0]
error = PageRedirect(self.code, location=location)
self.noPage(Failure(error))
return client.HTTPDownloader.gotHeaders(self, headers)
def request_callback(self, response):
finished = Deferred()
if not self.handle_redirect and response.code in (
http.MOVED_PERMANENTLY,
http.FOUND,
http.SEE_OTHER,
http.TEMPORARY_REDIRECT,
):
location = response.headers.getRawHeaders(b'location')[0]
error = PageRedirect(response.code, location=location)
finished.errback(Failure(error))
else:
headers = response.headers
body_length = int(
headers.getRawHeaders(b'content-length', default=[0])[0])
if headers.hasHeader(
b'content-disposition') and not self.force_filename:
content_disp = headers.getRawHeaders(
b'content-disposition')[0].decode('utf-8')
content_disp_params = cgi.parse_header(content_disp)[1]
if 'filename' in content_disp_params:
new_file_name = content_disp_params['filename']
new_file_name = sanitise_filename(new_file_name)
new_file_name = os.path.join(
os.path.split(self.filename)[0], new_file_name)
count = 1
fileroot = os.path.splitext(new_file_name)[0]
fileext = os.path.splitext(new_file_name)[1]
while os.path.isfile(new_file_name):
# Increment filename if already exists
new_file_name = '%s-%s%s' % (fileroot, count, fileext)
count += 1
self.filename = new_file_name
cont_type_header = headers.getRawHeaders(
b'content-type')[0].decode()
cont_type, params = cgi.parse_header(cont_type_header)
# Only re-ecode text content types.
encoding = None
if cont_type.startswith('text/'):
encoding = params.get('charset', None)
response.deliverBody(
BodyHandler(response.request, finished, body_length, self,
encoding))
return finished
class HTTPDownloader(client.HTTPDownloader):
"""
Factory class for downloading files and keeping track of progress.
"""
def __init__(self,
url,
filename,
part_callback=None,
headers=None,
force_filename=False,
allow_compression=True):
"""
:param url: the url to download from
:type url: string
:param filename: the filename to save the file as
:type filename: string
:param part_callback: a function to be called when a part of data
is received, it's signature should be: func(data, current_length, total_length)
:type part_callback: function
:param headers: any optional headers to send
:type headers: dictionary
"""
self.part_callback = part_callback
self.current_length = 0
self.decoder = None
self.value = filename
self.force_filename = force_filename
self.allow_compression = allow_compression
agent = "Deluge/%s (http://deluge-torrent.org)" % get_version()
client.HTTPDownloader.__init__(self,
url,
filename,
headers=headers,
agent=agent)
def gotStatus(self, version, status, message):
self.code = int(status)
client.HTTPDownloader.gotStatus(self, version, status, message)
def gotHeaders(self, headers):
if self.code == http.OK:
if "content-length" in headers:
self.total_length = int(headers["content-length"][0])
else:
self.total_length = 0
if self.allow_compression and "content-encoding" in headers and \
headers["content-encoding"][0] in ("gzip", "x-gzip", "deflate"):
# Adding 32 to the wbits enables gzip & zlib decoding (with automatic header detection)
# Adding 16 just enables gzip decoding (no zlib)
self.decoder = zlib.decompressobj(zlib.MAX_WBITS + 32)
if "content-disposition" in headers and not self.force_filename:
try:
new_file_name = str(headers["content-disposition"]
[0]).split(";")[1].split("=")[1]
new_file_name = sanitise_filename(new_file_name)
new_file_name = os.path.join(
os.path.split(self.fileName)[0], new_file_name)
except Exception, e:
log.exception(e)
else:
self.fileName = new_file_name
self.value = new_file_name
elif self.code in (http.MOVED_PERMANENTLY, http.FOUND, http.SEE_OTHER,
http.TEMPORARY_REDIRECT):
location = headers["location"][0]
error = PageRedirect(self.code, location=location)
self.noPage(Failure(error))
