def on_download_fail(result): if result.check(twisted.web.error.PageRedirect): new_url = urljoin(url, result.getErrorMessage().split(" to ")[1]) result = httpdownloader.download_file(new_url, tmp_file, headers=headers) result.addCallbacks(on_download_success, on_download_fail) elif result.check(twisted.web.client.PartialDownloadError): result = httpdownloader.download_file(url, tmp_file, headers=headers, allow_compression=False) result.addCallbacks(on_download_success, on_download_fail) else: log.error("Error occurred downloading torrent from %s", url) log.error("Reason: %s", result.getErrorMessage()) return result
def on_download_fail(failure): if failure.check(twisted.web.error.PageRedirect): new_url = urljoin(url, failure.getErrorMessage().split(" to ")[1]) result = download_file(new_url, tempfile.mkstemp()[1], headers=headers, force_filename=True) result.addCallbacks(on_download_success, on_download_fail) elif failure.check(twisted.web.client.PartialDownloadError): result = download_file(url, tempfile.mkstemp()[1], headers=headers, force_filename=True, allow_compression=False) result.addCallbacks(on_download_success, on_download_fail) else: # Log the error and pass the failure onto the client log.error("Error occurred downloading torrent from %s", url) log.error("Reason: %s", failure.getErrorMessage()) result = failure return result
def on_download_fail(failure): if failure.check(twisted.web.error.PageRedirect): new_url = urljoin(url, failure.getErrorMessage().split(" to ")[1]) result = download_file(new_url, tempfile.mkstemp()[1], headers=headers, force_filename=True) result.addCallbacks(on_download_success, on_download_fail) elif failure.check(twisted.web.client.PartialDownloadError): result = download_file(url, tempfile.mkstemp()[1], headers=headers, force_filename=True, allow_compression=False) result.addCallbacks(on_download_success, on_download_fail) else: # Log the error and pass the failure onto the client log.error("Error occured downloading torrent from %s", url) log.error("Reason: %s", failure.getErrorMessage()) result = failure return result
def _get_filename(self, host): candidate_urls = [] try: url = yield _FaviconClient("http://%s/" % host).execute() if url: candidate_urls.append(url) except Exception: log.debug("", exc_info=True) candidate_urls.append("http://%s/favicon.ico" % host) for url in candidate_urls: filename = os.path.join(self.image_dir, host + os.path.splitext(url)[1]) try: yield httpdownloader.download_file(url, filename, force_filename=True) except Exception: log.debug("", exc_info=True) else: break else: filename = None self.images[host] = filename for d in self._waiting[host]: d.callback(filename) del self._waiting[host]
def test_download_with_rename_exists(self): open('renamed', 'w').close() url = "http://deluge-torrent.org/httpdownloader.php?test=rename&filename=renamed" d = download_file(url, "original") d.addCallback(self.assertEqual, "renamed-1") d.addCallback(self.assertContains, "This file should be called renamed") return d
def test_download_with_required_cookies(self): url = "http://deluge-torrent.org/httpdownloader.php?test=cookie" cookie = { "cookie" : "password=deluge" } d = download_file(url, "monster", headers=cookie) d.addCallback(self.assertEqual, "monster") d.addCallback(self.assertContains, "COOKIE MONSTER!") return d
def download_torrent_from_url(self, url, cookie=None): """ Download a torrent file from a url to a temporary directory. :param url: the url of the torrent :type url: string :returns: the temporary file name of the torrent file :rtype: string """ def on_download_success(result): log.debug('Successfully downloaded %s to %s', url, result) return result def on_download_fail(result): log.error('Failed to add torrent from url %s', url) return result tempdir = tempfile.mkdtemp(prefix='delugeweb-') tmp_file = os.path.join(tempdir, url.split('/')[-1]) log.debug('filename: %s', tmp_file) headers = {} if cookie: headers['Cookie'] = cookie log.debug('cookie: %s', cookie) d = httpdownloader.download_file(url, tmp_file, headers=headers) d.addCallbacks(on_download_success, on_download_fail) return d
def test_download_with_required_cookies(self): url = self.get_url('cookie') cookie = {'cookie': 'password=deluge'} d = download_file(url, fname('monster'), headers=cookie) d.addCallback(self.assertEqual, fname('monster')) d.addCallback(self.assertContains, 'COOKIE MONSTER!') return d
def download_list(self, url=None): """ Downloads the blocklist specified by 'url' in the config :param url: optional url to download from, defaults to config value :type url: string :returns: a Deferred which fires once the blocklist has been downloaded :rtype: Deferred """ def on_retrieve_data(data, current_length, total_length): if total_length: fp = float(current_length) / total_length if fp > 1.0: fp = 1.0 else: fp = 0.0 self.file_progress = fp import socket socket.setdefaulttimeout(self.config["timeout"]) if not url: url = self.config["url"] headers = {} if self.config["last_update"] and not self.force_download: headers['If-Modified-Since'] = formatdate(self.config["last_update"], usegmt=True) log.debug("Attempting to download blocklist %s", url) log.debug("Sending headers: %s", headers) self.is_downloading = True return download_file(url, deluge.configmanager.get_config_dir("blocklist.download"), on_retrieve_data, headers)
def download_list(self, url=None): """ Downloads the blocklist specified by 'url' in the config :param url: optional url to download from, defaults to config value :type url: string :returns: a Deferred which fires once the blocklist has been downloaded :rtype: Deferred """ def on_retrieve_data(data, current_length, total_length): if total_length: fp = float(current_length) / total_length if fp > 1.0: fp = 1.0 else: fp = 0.0 self.file_progress = fp import socket socket.setdefaulttimeout(self.config["timeout"]) if not url: url = self.config["url"] headers = {} if self.config["last_update"] and not self.force_download: headers["If-Modified-Since"] = formatdate(self.config["last_update"], usegmt=True) log.debug("Attempting to download blocklist %s", url) log.debug("Sending headers: %s", headers) self.is_downloading = True return download_file(url, deluge.configmanager.get_config_dir("blocklist.download"), on_retrieve_data, headers)
def download_torrent_from_url(self, url, cookie=None): """ Download a torrent file from a URL to a temporary directory. :param url: the URL of the torrent :type url: string :returns: the temporary file name of the torrent file :rtype: string """ def on_download_success(result): log.debug('Successfully downloaded %s to %s', url, result) return result def on_download_fail(result): log.error('Failed to add torrent from url %s', url) return result tempdir = tempfile.mkdtemp(prefix='delugeweb-') tmp_file = os.path.join(tempdir, url.split('/')[-1]) log.debug('filename: %s', tmp_file) headers = {} if cookie: headers['Cookie'] = cookie log.debug('cookie: %s', cookie) d = httpdownloader.download_file(url, tmp_file, headers=headers) d.addCallbacks(on_download_success, on_download_fail) return d
def test_download_with_rename_exists(self): open('renamed', 'w').close() url = "http://localhost:51242/rename?filename=renamed" d = download_file(url, "original") d.addCallback(self.assertEqual, "renamed-1") d.addCallback(self.assertContains, "This file should be called renamed") return d
def test_download_with_rename_sanitised(self): url = "http://localhost:51242/rename?filename=/etc/passwd" d = download_file(url, "original") d.addCallback(self.assertEqual, "passwd") d.addCallback(self.assertContains, "This file should be called /etc/passwd") return d
def test_download_with_rename_exists(self): open(fname('renamed'), 'w').close() url = self.get_url('rename?filename=renamed') d = download_file(url, fname('original')) d.addCallback(self.assertEqual, fname('renamed-1')) d.addCallback(self.assertContains, 'This file should be called renamed') return d
def test_download_with_required_cookies(self): url = "http://localhost:51242/cookie" cookie = {"cookie": "password=deluge"} d = download_file(url, "monster", headers=cookie) d.addCallback(self.assertEqual, "monster") d.addCallback(self.assertContains, "COOKIE MONSTER!") return d
def test_download_with_required_cookies(self): url = "http://localhost:51242/cookie" cookie = { "cookie" : "password=deluge" } d = download_file(url, "monster", headers=cookie) d.addCallback(self.assertEqual, "monster") d.addCallback(self.assertContains, "COOKIE MONSTER!") return d
def test_download_with_rename_sanitised(self): url = self.get_url('rename?filename=/etc/passwd') d = download_file(url, fname('original')) d.addCallback(self.assertEqual, fname('passwd')) d.addCallback(self.assertContains, 'This file should be called /etc/passwd') return d
def test_page_not_modified(self): headers = {'If-Modified-Since': formatdate(usegmt=True)} d = download_file("http://localhost:51242/", "index.html", headers=headers) d.addCallback(self.fail) d.addErrback(self.assertIsInstance, Failure) return d
def test_page_redirect_unhandled(self): url = self.get_url('redirect') d = download_file(url, fname('none')) d.addCallback(self.fail) def on_redirect(failure): self.assertTrue(type(failure), PageRedirect) d.addErrback(on_redirect) return d
def test_download_with_gzip_encoding_disabled(self): url = self.get_url('gzip?msg=fail') d = download_file(url, fname('gzip_encoded'), allow_compression=False) def cb(result): print(result) d.addCallback(self.assertNotContains, b'fail', file_mode='rb') return d
def test_download_binary_ignore_charset(self): """Ignore charset for binary content-type header e.g. torrent files""" url = self.get_url('torrent') headers = {'content-charset': 'Windows-1251'} filepath = fname('test.torrent') d = download_file(url, fname('test.torrent'), headers=headers) d.addCallback(self.assertEqual, filepath) d.addCallback(self.assertContains, 'Binary attachment ignore charset 世丕且\n') return d
def test_download_with_rename_sanitised(self): if windows_check(): raise unittest.SkipTest('on windows \\ != / for path names') url = self.get_url('rename?filename=/etc/passwd') d = download_file(url, fname('original')) d.addCallback(self.assertEqual, fname('passwd')) d.addCallback(self.assertContains, 'This file should be called /etc/passwd') return d
def test_download_with_rename_exists(self): if windows_check(): raise unittest.SkipTest('on windows \\ != / for path names') open(fname('renamed'), 'w').close() url = self.get_url('rename?filename=renamed') d = download_file(url, fname('original')) d.addCallback(self.assertEqual, fname('renamed-1')) d.addCallback(self.assertContains, 'This file should be called renamed') return d
def test_download_text_reencode_charset(self): """Re-encode as UTF-8 specified charset for text content-type header""" url = self.get_url('attachment') filepath = fname('test.txt') headers = { 'content-charset': 'Windows-1251', 'content-append': 'бвгде' } d = download_file(url, filepath, headers=headers) d.addCallback(self.assertEqual, filepath) d.addCallback(self.assertContains, 'Attachment with no filename setбвгде') return d
def add_from_url(self, url): dialog = gtk.Dialog( _('Downloading...'), flags=gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT | gtk.DIALOG_NO_SEPARATOR, parent=self.dialog, ) dialog.set_transient_for(self.dialog) pb = gtk.ProgressBar() dialog.vbox.pack_start(pb, True, True, 0) dialog.show_all() # Create a tmp file path import tempfile tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') def on_part(data, current_length, total_length): if total_length: percent = current_length / total_length pb.set_fraction(percent) pb.set_text('%.2f%% (%s / %s)' % ( percent * 100, deluge.common.fsize(current_length), deluge.common.fsize(total_length), )) else: pb.pulse() pb.set_text('%s' % deluge.common.fsize(current_length)) def on_download_success(result): self.add_from_files([result]) dialog.destroy() def on_download_fail(result): log.debug('Download failed: %s', result) dialog.destroy() ErrorDialog( _('Download Failed'), '%s %s' % (_('Failed to download:'), url), details=result.getErrorMessage(), parent=self.dialog, ).run() return result d = download_file(url, tmp_file, on_part) os.close(tmp_fd) d.addCallbacks(on_download_success, on_download_fail)
def download_page(self, host, url=None): """ Downloads a tracker host's page If no url is provided, it bases the url on the host :param host: the tracker host :type host: string :param url: the (optional) url of the host :type url: string :returns: the filename of the tracker host's page :rtype: Deferred """ if not url: url = self.host_to_url(host) log.debug("Downloading %s %s", host, url) return download_file(url, mkstemp()[1], force_filename=True)
def download_page(self, host, url=None): """ Downloads a tracker host's page If no url is provided, it bases the url on the host :param host: the tracker host :type host: string :param url: the (optional) url of the host :type url: string :returns: the filename of the tracker host's page :rtype: Deferred """ if not url: url = self.host_to_url(host) log.debug('Downloading %s %s', host, url) tmp_fd, tmp_file = mkstemp(prefix='deluge_ticon.') os.close(tmp_fd) return download_file(url, tmp_file, force_filename=True, handle_redirects=False)
def add_from_url(self, url): dialog = gtk.Dialog( _('Downloading...'), flags=gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT | gtk.DIALOG_NO_SEPARATOR, parent=self.dialog) dialog.set_transient_for(self.dialog) pb = gtk.ProgressBar() dialog.vbox.pack_start(pb, True, True, 0) dialog.show_all() # Create a tmp file path import tempfile tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') def on_part(data, current_length, total_length): if total_length: percent = current_length / total_length pb.set_fraction(percent) pb.set_text('%.2f%% (%s / %s)' % ( percent * 100, deluge.common.fsize(current_length), deluge.common.fsize(total_length))) else: pb.pulse() pb.set_text('%s' % deluge.common.fsize(current_length)) def on_download_success(result): self.add_from_files([result]) dialog.destroy() def on_download_fail(result): log.debug('Download failed: %s', result) dialog.destroy() ErrorDialog( _('Download Failed'), '%s %s' % (_('Failed to download:'), url), details=result.getErrorMessage(), parent=self.dialog ).run() return result d = download_file(url, tmp_file, on_part) os.close(tmp_fd) d.addCallbacks(on_download_success, on_download_fail)
def download_torrent_from_url(self, url, cookie=None): """ Download a torrent file from a url to a temporary directory. :param url: the url of the torrent :type url: string :returns: the temporary file name of the torrent file :rtype: string """ def on_download_success(result): log.debug("Successfully downloaded %s to %s", url, result) return result def on_download_fail(result): if result.check(twisted.web.error.PageRedirect): new_url = urljoin(url, result.getErrorMessage().split(" to ")[1]) result = httpdownloader.download_file(new_url, tmp_file, headers=headers) result.addCallbacks(on_download_success, on_download_fail) elif result.check(twisted.web.client.PartialDownloadError): result = httpdownloader.download_file(url, tmp_file, headers=headers, allow_compression=False) result.addCallbacks(on_download_success, on_download_fail) else: log.error("Error occured downloading torrent from %s", url) log.error("Reason: %s", result.getErrorMessage()) return result tempdir = tempfile.mkdtemp(prefix="delugeweb-") tmp_file = os.path.join(tempdir, url.split("/")[-1]) log.debug("filename: %s", tmp_file) headers = {} if cookie: headers["Cookie"] = cookie log.debug("cookie: %s", cookie) d = httpdownloader.download_file(url, tmp_file, headers=headers) d.addCallbacks(on_download_success, on_download_fail) return d
def download_icon(self, icons, host): """ Downloads the first available icon from icons :param icons: a list of icons :type icons: list :param host: the tracker's host name :type host: string :returns: a Deferred which fires with the downloaded icon's filename :rtype: Deferred """ if len(icons) == 0: raise NoIconsError, "empty icons list" (url, mimetype) = icons.pop(0) d = download_file(url, os.path.join(self.dir, host_to_icon_name(host, mimetype)), force_filename=True) d.addCallback(self.check_icon_is_valid) if icons: d.addErrback(self.on_download_icon_fail, host, icons) return d
def download_list(self, url=None): """Downloads the blocklist specified by 'url' in the config. Args: url (str, optional): url to download from, defaults to config value. Returns: Deferred: a Deferred which fires once the blocklist has been downloaded. """ def on_retrieve_data(data, current_length, total_length): if total_length: fp = current_length / total_length if fp > 1.0: fp = 1.0 else: fp = 0.0 self.file_progress = fp import socket socket.setdefaulttimeout(self.config['timeout']) if not url: url = self.config['url'] headers = {} if self.config['last_update'] and not self.force_download: headers['If-Modified-Since'] = formatdate( self.config['last_update'], usegmt=True) log.debug('Attempting to download blocklist %s', url) log.debug('Sending headers: %s', headers) self.is_downloading = True return download_file( url, deluge.configmanager.get_config_dir('blocklist.download'), on_retrieve_data, headers, )
def download_torrent_from_url(self, url, cookie=None): """ Download a torrent file from a url to a temporary directory. :param url: the url of the torrent :type url: string :returns: the temporary file name of the torrent file :rtype: string """ def on_download_success(result): log.debug("Successfully downloaded %s to %s", url, result) return result def on_download_fail(result): if result.check(twisted.web.error.PageRedirect): new_url = urljoin(url, result.getErrorMessage().split(" to ")[1]) result = httpdownloader.download_file(new_url, tmp_file, headers=headers) result.addCallbacks(on_download_success, on_download_fail) elif result.check(twisted.web.client.PartialDownloadError): result = httpdownloader.download_file(url, tmp_file, headers=headers, allow_compression=False) result.addCallbacks(on_download_success, on_download_fail) else: log.error("Error occurred downloading torrent from %s", url) log.error("Reason: %s", result.getErrorMessage()) return result tempdir = tempfile.mkdtemp(prefix="delugeweb-") tmp_file = os.path.join(tempdir, url.split("/")[-1]) log.debug("filename: %s", tmp_file) headers = {} if cookie: headers["Cookie"] = cookie log.debug("cookie: %s", cookie) d = httpdownloader.download_file(url, tmp_file, headers=headers) d.addCallbacks(on_download_success, on_download_fail) return d
def add_torrent_url(self, url, options, headers=None): """ Adds a torrent from a url. Deluge will attempt to fetch the torrent from url prior to adding it to the session. :param url: the url pointing to the torrent file :type url: string :param options: the options to apply to the torrent on add :type options: dict :param headers: any optional headers to send :type headers: dict :returns: a Deferred which returns the torrent_id as a str or None """ log.info('Attempting to add url %s', url) def on_download_success(filename): # We got the file, so add it to the session with open(filename, 'rb') as _file: data = _file.read() try: os.remove(filename) except OSError as ex: log.warning('Could not remove temp file: %s', ex) return self.add_torrent_file(filename, b64encode(data), options) def on_download_fail(failure): # Log the error and pass the failure onto the client log.error('Failed to add torrent from url %s', url) return failure tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') os.close(tmp_fd) d = download_file(url, tmp_file, headers=headers, force_filename=True) d.addCallbacks(on_download_success, on_download_fail) return d
def download_list(self, url=None): """Downloads the blocklist specified by 'url' in the config. Args: url (str, optional): url to download from, defaults to config value. Returns: Deferred: a Deferred which fires once the blocklist has been downloaded. """ def on_retrieve_data(data, current_length, total_length): if total_length: fp = current_length / total_length if fp > 1.0: fp = 1.0 else: fp = 0.0 self.file_progress = fp import socket socket.setdefaulttimeout(self.config['timeout']) if not url: url = self.config['url'] headers = {} if self.config['last_update'] and not self.force_download: headers['If-Modified-Since'] = formatdate(self.config['last_update'], usegmt=True) log.debug('Attempting to download blocklist %s', url) log.debug('Sending headers: %s', headers) self.is_downloading = True return download_file( url, deluge.configmanager.get_config_dir('blocklist.download'), on_retrieve_data, headers )
def add_torrent_url(self, url, options, headers=None): """ Adds a torrent from a url. Deluge will attempt to fetch the torrent from url prior to adding it to the session. :param url: the url pointing to the torrent file :type url: string :param options: the options to apply to the torrent on add :type options: dict :param headers: any optional headers to send :type headers: dict :returns: a Deferred which returns the torrent_id as a str or None """ log.info('Attempting to add url %s', url) def on_download_success(filename): # We got the file, so add it to the session with open(filename, 'rb') as _file: data = _file.read() try: os.remove(filename) except OSError as ex: log.warning('Could not remove temp file: %s', ex) return self.add_torrent_file(filename, base64.encodestring(data), options) def on_download_fail(failure): # Log the error and pass the failure onto the client log.error('Failed to add torrent from url %s', url) return failure tmp_fd, tmp_file = tempfile.mkstemp(prefix='deluge_url.', suffix='.torrent') os.close(tmp_fd) d = download_file(url, tmp_file, headers=headers, force_filename=True) d.addCallbacks(on_download_success, on_download_fail) return d
def test_page_not_modified(self): headers = {'If-Modified-Since': formatdate(usegmt=True)} d = download_file(self.get_url(), fname('index.html'), headers=headers) d.addCallback(self.fail) d.addErrback(self.assertIsInstance, Failure) return d
def test_page_redirect(self): url = self.get_url('redirect') d = download_file(url, fname('none'), handle_redirects=True) d.addCallback(self.assertEqual, fname('none')) d.addErrback(self.fail) return d
def test_page_not_found(self): d = download_file(self.get_url('page/not/found'), fname('none')) d.addCallback(self.fail) d.addErrback(self.assertIsInstance, Failure) return d
def test_download_with_gzip_encoding_disabled(self): url = self.get_url('gzip?msg=fail') d = download_file(url, fname('gzip_encoded'), allow_compression=False) d.addCallback(self.assertNotContains, b'fail') return d
def test_page_not_modified(self): headers = { 'If-Modified-Since' : formatdate(usegmt=True) } d = download_file("http://localhost:51242/", "index.html", headers=headers) d.addCallback(self.fail) d.addErrback(self.assertIsInstance, Failure) return d
def test_download_with_rename_prevented(self): url = self.get_url('rename?filename=spam') d = download_file(url, fname('forced'), force_filename=True) d.addCallback(self.assertEqual, fname('forced')) d.addCallback(self.assertContains, 'This file should be called spam') return d
def test_download_with_gzip_encoding(self): url = self.get_url('gzip?msg=success') d = download_file(url, fname('gzip_encoded')) d.addCallback(self.assertContains, b'success') return d
if failure.check(twisted.web.error.PageRedirect): new_url = urljoin(url, failure.getErrorMessage().split(" to ")[1]) result = download_file(new_url, tempfile.mkstemp()[1], headers=headers, force_filename=True) result.addCallbacks(on_download_success, on_download_fail) elif failure.check(twisted.web.client.PartialDownloadError): result = download_file(url, tempfile.mkstemp()[1], headers=headers, force_filename=True, allow_compression=False) result.addCallbacks(on_download_success, on_download_fail) else: # Log the error and pass the failure onto the client log.error("Error occured downloading torrent from %s", url) log.error("Reason: %s", failure.getErrorMessage()) result = failure return result d = download_file(url, tempfile.mkstemp()[1], headers=headers, force_filename=True) d.addCallbacks(on_download_success, on_download_fail) return d @export def add_torrent_magnet(self, uri, options): """ Adds a torrent from a magnet link. :param uri: the magnet link :type uri: string :param options: the options to apply to the torrent on add :type options: dict :returns: the torrent_id :rtype: string
def test_download_without_required_cookies(self): url = self.get_url('cookie') d = download_file(url, fname('none')) d.addCallback(self.fail) d.addErrback(self.assertIsInstance, Failure) return d
def test_download(self): d = download_file(self.get_url(), fname('index.html')) d.addCallback(self.assertEqual, fname('index.html')) return d