def _do_download_request(self,
document,
callback=None,
errback=None,
**params):
url = '%s/%s?%s' % (self.API_SERVER, document,
urllib.urlencode(params))
headers = self._get_headers()
file_ = tempfile.NamedTemporaryFile(delete=False)
downloader = HTTPDownloader(url,
file_.name,
agent=headers['User-Agent'],
headers=headers)
def errback_(error):
if errback is not None:
code = getattr(downloader, 'status', None)
errback(code, error)
def callback_(res):
if getattr(downloader, 'status', None) == '200' and callback:
callback(file_.name)
# Remove the temporary file after the callback has handled it
os.remove(file_.name)
downloader.deferred.addErrback(errback_)
downloader.deferred.addCallback(callback_)
parsed = urlparse.urlparse(url)
reactor.connectTCP(
parsed.netloc.split(':')[0], parsed.port or 80, downloader)
return downloader.deferred
def download(url, file, contextFactory = None, *args, **kwargs):
"""Download a remote file from http(s) or ftp.
@param file: path to file on filesystem, or file-like object.
See HTTPDownloader to see what extra args can be passed if remote file
is accessible via http or https. Both Backends should offer supportPartial.
"""
scheme, host, port, path, username, password = _parse(url)
if scheme == 'ftp':
if not (username and password):
username = '******'
password = '******'
client = FTPDownloader(
host,
port,
path,
file,
username,
password,
*args,
**kwargs
)
return client.deferred
# We force username and password here as we lack a satisfying input method
if username and password:
from base64 import encodestring
# twisted will crash if we don't rewrite this ;-)
url = scheme + '://' + host + ':' + str(port) + path
basicAuth = encodestring("%s:%s" % (username, password))
authHeader = "Basic " + basicAuth.strip()
AuthHeaders = {"Authorization": authHeader}
if kwargs.has_key("headers"):
kwargs["headers"].update(AuthHeaders)
else:
kwargs["headers"] = AuthHeaders
factory = HTTPDownloader(url, file, *args, **kwargs)
if scheme == 'https':
from twisted.internet import ssl
if contextFactory is None:
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def downloadHTTP(url, fileOrName):
"""
Another method to download from HTTP host.
"""
global _UserAgentString
scheme, host, port, path = parse_url(url)
factory = HTTPDownloader(url, fileOrName, agent=_UserAgentString)
if proxy_is_on():
host = get_proxy_host()
port = get_proxy_port()
factory.path = url
reactor.connectTCP(host, port, factory)
return factory.deferred
def _command_download(self, data):
reactor = self._context["reactor"]
session_files = self._context["session_files"]
audio_id = data["audio_id"]
partial_url = data["partial_url"]
ip_address = str(self.transport.getPeer().host)
url = "http://" + ip_address + partial_url
file_path = session_files.session_dir / f"{audio_id}.opus"
log.info(f"Downloading file from {url} to {file_path}")
url_bytes = url.encode("utf-8")
url_parsed = URI.fromBytes(url_bytes)
factory = HTTPDownloader(url_bytes, str(file_path))
reactor.connectTCP(url_parsed.host, url_parsed.port, factory)
d = factory.deferred
def on_success(data):
# File downloaded succesfully, tell the server
result = {
"command": "update_downloaded",
"audio_id": audio_id,
"result": "success"
}
result_json = json.dumps(result)
self._tcp_packetizer.write(result_json)
def on_error(error):
# File failed to downloaded succesfully, tell the server
log.error(f"Failed to download file at '{url}': {error}")
result = {
"command": "update_downloaded",
"audio_id": audio_id,
"result": "failure",
"error": str(error)
}
result_json = json.dumps(result)
self._tcp_packetizer.write(result_json)
d.addCallback(on_success)
d.addErrback(on_error)
return d
def downloadSSL(url, fileOrName, progress_func, certificates_filenames):
"""
Another method to download from HTTPS.
Not used at the moment.
"""
global _UserAgentString
from twisted.internet import ssl
from OpenSSL import SSL # @UnresolvedImport
class MyClientContextFactory(ssl.ClientContextFactory):
def __init__(self, certificates_filenames):
self.certificates_filenames = list(certificates_filenames)
def verify(self, connection, x509, errnum, errdepth, ok):
return ok
def getContext(self):
ctx = ssl.ClientContextFactory.getContext(self)
for cert in self.certificates_filenames:
try:
ctx.load_verify_locations(cert)
except:
pass
ctx.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
self.verify)
return ctx
scheme, host, port, path = parse_url(url)
if not isinstance(certificates_filenames, list):
certificates_filenames = [
certificates_filenames,
]
cert_found = False
for cert in certificates_filenames:
if os.path.isfile(cert) and os.access(cert, os.R_OK):
cert_found = True
break
if not cert_found:
return fail(Exception('no one ssl certificate found'))
factory = HTTPDownloader(url, fileOrName, agent=_UserAgentString)
contextFactory = MyClientContextFactory(certificates_filenames)
reactor.connectSSL(host, port, factory,
contextFactory) # @UndefinedVariable
return factory.deferred
def start(self):
g_logger.info("URL: %s" % self._url)
g_logger.info("Local Path: %s" % self._local_path)
g_logger.info("Download started.")
print "test"
factory = HTTPDownloader(self._url, self._local_path)
factory.deferred.addErrback(g_logger.error)
factory.deferred.addCallback(self._downloadComplete)
if hasattr(self, '_callback'):
factory.deferred.addCallback(self._callback, *self._callback_args,
**self._callback_kw)
if self._url.startswith("https://"):
from twisted.internet import ssl
content_factory = ssl.ClientContextFactory()
reactor.connectSSL(factory.host, factory.port, factory, content_factory, timeout, bindAddress)
else:
reactor.connectTCP(factory.host, factory.port, factory)
def downloadSSL(url, fileOrName, progress_func, certificates_filenames):
"""
Another method to download from HTTPS.
"""
global _UserAgentString
scheme, host, port, path = parse_url(url)
if not isinstance(certificates_filenames, types.ListType):
certificates_filenames = [
certificates_filenames,
]
cert_found = False
for cert in certificates_filenames:
if os.path.isfile(cert) and os.access(cert, os.R_OK):
cert_found = True
break
if not cert_found:
return fail(Exception('no one ssl certificate found'))
factory = HTTPDownloader(url, fileOrName, agent=_UserAgentString)
contextFactory = MyClientContextFactory(certificates_filenames)
reactor.connectSSL(host, port, factory, contextFactory)
return factory.deferred
