def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError, ('ftp error', 'no host given')
# XXX handle custom username & password
host = socket.gethostbyname(host)
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
path, attrs = splitattr(req.get_selector())
path = unquote(path)
dirs = string.splitfields(path, '/')
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
user = passwd = '' # XXX
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitattr(attr)
if string.lower(attr) == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = string.upper(value)
fp, retrlen = fw.retrfile(file, type)
if retrlen is not None and retrlen >= 0:
sf = StringIO('Content-Length: %d\n' % retrlen)
headers = mimetools.Message(sf)
else:
headers = noheaders()
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors, msg:
raise IOError, ('ftp error', msg), sys.exc_info()[2]
class FTPRangeHandler(urllib2.FTPHandler):
"""
FTP Range support..
"""
def ftp_open(self, req):
host = req.get_host()
host, port = urllib.splitport(host)
if port is None:
port = ftplib.FTP_PORT
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise FetchError(msg)
path, attrs = urllib.splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(urllib.unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp('', '', host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = urllib.splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
rawr = req.headers.get('Range', None)
if rawr:
rest = int(rawr.split("=")[1].rstrip("-"))
else:
rest = 0
fp, retrlen = fw.retrfile(file, type, rest)
fb, lb = rest, retrlen
if retrlen is None or retrlen == 0:
raise RangeError
retrlen = lb - fb
if retrlen < 0:
# beginning of range is larger than file
raise RangeError
headers = ''
mtype = guess_type(req.get_full_url())[0]
if mtype:
headers += 'Content-Type: %s\n' % mtype
if retrlen is not None and retrlen >= 0:
headers += 'Content-Length: %d\n' % retrlen
try:
from cStringIO import StringIO
except ImportError, msg:
from StringIO import StringIO
return urllib.addinfourl(fp, Message(StringIO(headers)),
req.get_full_url())
class FTPHandler(BaseHandler):
def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError, ('ftp error', 'no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
fp, retrlen = fw.retrfile(file, type)
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors, msg:
raise IOError, ('ftp error', msg), sys.exc_info()[2]
def start(self, destfile=None, destfd=None):
urllib._urlopener = OLPCURLopener()
self._info = urllib.urlopen(self._url)
self._outf = None
self._fname = None
if destfd and not destfile:
raise ValueError('Must provide destination file too when'
' specifying file descriptor')
if destfile:
self._suggested_fname = os.path.basename(destfile)
self._fname = os.path.abspath(os.path.expanduser(destfile))
if destfd:
# Use the user-supplied destination file descriptor
self._outf = destfd
else:
self._outf = os.open(self._fname, os.O_RDWR |
os.O_TRUNC | os.O_CREAT, 0644)
else:
fname = self._get_filename_from_headers(self._info.headers)
self._suggested_fname = fname
garbage_, path = urllib.splittype(self._url)
garbage_, path = urllib.splithost(path or "")
path, garbage_ = urllib.splitquery(path or "")
path, garbage_ = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
(self._outf, self._fname) = tempfile.mkstemp(suffix=suffix,
dir=self._destdir)
fcntl.fcntl(self._info.fp.fileno(), fcntl.F_SETFD, os.O_NDELAY)
self._srcid = GObject.io_add_watch(self._info.fp.fileno(),
GObject.IO_IN | GObject.IO_ERR,
self._read_next_chunk)
def start(self, destfile=None, destfd=None):
self._info = urllib.urlopen(self._url)
self._outf = None
self._fname = None
if destfd and not destfile:
raise ValueError('Must provide destination file too when'
' specifying file descriptor')
if destfile:
self._suggested_fname = os.path.basename(destfile)
self._fname = os.path.abspath(os.path.expanduser(destfile))
if destfd:
# Use the user-supplied destination file descriptor
self._outf = destfd
else:
self._outf = os.open(self._fname, os.O_RDWR |
os.O_TRUNC | os.O_CREAT, 0644)
else:
fname = self._get_filename_from_headers(self._info.headers)
self._suggested_fname = fname
garbage_, path = urllib.splittype(self._url)
garbage_, path = urllib.splithost(path or "")
path, garbage_ = urllib.splitquery(path or "")
path, garbage_ = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
(self._outf, self._fname) = tempfile.mkstemp(suffix=suffix,
dir=self._destdir)
fcntl.fcntl(self._info.fp.fileno(), fcntl.F_SETFD, os.O_NDELAY)
self._srcid = GLib.io_add_watch(self._info.fp.fileno(),
GLib.IO_IN | GLib.IO_ERR,
self._read_next_chunk)
def _splitsuffix(self, url):
'Split the suffix off of a url.'
garbage, pathpart = urllib.splittype(url)
garbage, path = urllib.splithost(pathpart or '')
pathpart, garbage = urllib.splitquery(pathpart or '')
pathpart, garbage = urllib.splitattr(pathpart or '')
return os.path.splitext(pathpart)[1]
def open_ftp(self, url):
host, path = urllib.splithost(url)
if not host: raise IOError, ('ftp error', 'no host given')
host, port = urllib.splitport(host)
user, host = urllib.splituser(host)
# if user: user, passwd = splitpasswd(user)
if user: passwd = getpass.getpass()
else: passwd = None
host = urllib.unquote(host)
user = urllib.unquote(user or '')
passwd = urllib.unquote(passwd or '')
host = socket.gethostbyname(host)
if not port:
import ftplib
port = ftplib.FTP_PORT
else:
port = int(port)
path, attrs = urllib.splitattr(path)
path = urllib.unquote(path)
dirs = string.splitfields(path, '/')
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]: dirs = dirs[1:]
key = (user, host, port, string.joinfields(dirs, '/'))
# XXX thread unsafe!
if len(self.ftpcache) > MAXFTPCACHE:
# Prune the cache, rather arbitrarily
for k in self.ftpcache.keys():
if k != key:
v = self.ftpcache[k]
del self.ftpcache[k]
v.close()
try:
if not self.ftpcache.has_key(key):
print 'Creating ftpwrapper: ',user,host,port,dirs
self.ftpcache[key] = \
urllib.ftpwrapper(user, passwd, host, port, dirs)
if not file: type = 'D'
else: type = 'I'
for attr in attrs:
attr, value = urllib.splitvalue(attr)
if string.lower(attr) == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = string.upper(value)
(fp, retrlen) = self.ftpcache[key].retrfile(file, type)
if retrlen is not None and retrlen >= 0:
import mimetools, StringIO
headers = mimetools.Message(StringIO.StringIO(
'Content-Length: %d\n' % retrlen))
else:
headers = noheaders()
return urllib.addinfourl(fp, headers, "ftp:" + url)
except urllib.ftperrors(), msg:
raise IOError, ('ftp error', msg), sys.exc_info()[2]
def open_ftp(self, url):
host, path = urllib.splithost(url)
if not host: raise IOError, ('ftp error', 'no host given')
host, port = urllib.splitport(host)
user, host = urllib.splituser(host)
# if user: user, passwd = splitpasswd(user)
if user: passwd = getpass.getpass()
else: passwd = None
host = urllib.unquote(host)
user = urllib.unquote(user or '')
passwd = urllib.unquote(passwd or '')
host = socket.gethostbyname(host)
if not port:
import ftplib
port = ftplib.FTP_PORT
else:
port = int(port)
path, attrs = urllib.splitattr(path)
path = urllib.unquote(path)
dirs = string.splitfields(path, '/')
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]: dirs = dirs[1:]
key = (user, host, port, string.joinfields(dirs, '/'))
# XXX thread unsafe!
if len(self.ftpcache) > MAXFTPCACHE:
# Prune the cache, rather arbitrarily
for k in self.ftpcache.keys():
if k != key:
v = self.ftpcache[k]
del self.ftpcache[k]
v.close()
try:
if not self.ftpcache.has_key(key):
print 'Creating ftpwrapper: ', user, host, port, dirs
self.ftpcache[key] = \
urllib.ftpwrapper(user, passwd, host, port, dirs)
if not file: type = 'D'
else: type = 'I'
for attr in attrs:
attr, value = urllib.splitvalue(attr)
if string.lower(attr) == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = string.upper(value)
(fp, retrlen) = self.ftpcache[key].retrfile(file, type)
if retrlen is not None and retrlen >= 0:
import mimetools, StringIO
headers = mimetools.Message(
StringIO.StringIO('Content-Length: %d\n' % retrlen))
else:
headers = noheaders()
return urllib.addinfourl(fp, headers, "ftp:" + url)
except urllib.ftperrors(), msg:
raise IOError, ('ftp error', msg), sys.exc_info()[2]
def ftp_open(self, req):
import ftplib
import mimetypes
host = req.get_host()
if not host:
raise URLError('ftp error: no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
passwd = passwd or ''
try:
host = socket.gethostbyname(host)
except socket.error as msg:
raise URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitvalue(attr)
if attr.lower() == 'type' and value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
fp, retrlen = fw.retrfile(file, type)
headers = ''
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += 'Content-type: %s\n' % mtype
if retrlen is not None and retrlen >= 0:
headers += 'Content-length: %d\n' % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors as msg:
raise URLError, 'ftp error: %s' % msg, sys.exc_info()[2]
return
def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError('ftp error', 'no host given')
# XXX handle custom username & password
try:
host = socket.gethostbyname(host)
except socket.error(msg):
raise URLError(msg)
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
path, attrs = splitattr(req.get_selector())
path = unquote(path)
dirs = path.split('/')
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
user = passwd = '' # XXX
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
fp, retrlen = fw.retrfile(file, type)
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-Type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors(msg):
raise IOError(('ftp error', msg), sys.exc_info()[2])
def relative_uri(uri, fn):
path, query = urllib.splitquery(uri)
path, attrs = urllib.splitattr(path)
if fn.startswith('/'):
scheme, _, path = path.partition(':')
host, path = urllib.splithost(path)
new_uri = '{}://{}/{}'.format(scheme, host, fn[1:])
else:
parent = path.rpartition('/')[0]
new_uri = '{}/{}'.format(parent, fn)
if attrs:
new_uri = '{};{}'.format(new_uri, ';'.join(attrs))
return new_uri
def __extract_video_id_from_uri(self, uri):
"""
GET uri like '/watch?v=AsXf9v¶m=1&p=3#junk'
RETURNS value for 'v' parameter --> 'AsXf9v'
"""
uri = uri.replace('&', ';')
uri = uri.replace('?', ';')
req, params = urllib.splitattr(uri)
for item in params:
k, v = urllib.splitvalue(params[0])
if k == 'v':
return v
raise ValueError("Can't find parameter 'v' from '%s'" % uri)
def __extract_video_id_from_uri(self, uri):
"""
GET uri like '/watch?v=AsXf9v¶m=1&p=3#junk'
RETURNS value for 'v' parameter --> 'AsXf9v'
"""
uri = uri.replace('&', ';')
uri = uri.replace('?', ';')
req, params = urllib.splitattr(uri)
for item in params:
k, v = urllib.splitvalue(params[0])
if k == 'v':
return v
raise ValueError("Can't find parameter 'v' from '%s'" % uri)
def addSubscriber(self, channel, session):
"""
Subscribe to a channel.
A subscribe request of /foo?a=b will listen to foo messages
if attribute a of the message has value "b".
"""
channel, attrs = urllib.splitquery(channel)
pattern = {}
if attrs:
attr, rest = urllib.splitattr(attrs)
for attr in [attr] + rest:
key, value = urllib.splitvalue(attr)
pattern[key] = value
self._subscribers.setdefault(channel, []).append((session, pattern))
def addSubscriber(self, channel, session):
"""
Subscribe to a channel.
A subscribe request of /foo?a=b will listen to foo messages
if attribute a of the message has value "b".
"""
channel, attrs = urllib.splitquery(channel)
pattern = {}
if attrs:
attr, rest = urllib.splitattr(attrs)
for attr in [attr] + rest:
key, value = urllib.splitvalue(attr)
pattern[key] = value
self._subscribers.setdefault(channel, []).append((session, pattern))
def rsync_rsh_open(self, req):
logger.debug('Opening rsync+rsh')
host = req.get_host()
if not host:
raise urllib2.URLError('rsync+ssh error: not host given')
if ':' in host:
raise urllib2.URLError(
'rsync+ssh error: \':\' character not supported in host')
path, attrs = urllib.splitattr(req.get_selector())
if not path:
raise urllib2.URLError('rsync+ssh error: no path given')
source = '{}:{}'.format(host, path)
return self.do_rsync(source, attrs=attrs)
def parse_handle(hdl):
"""Parse off options from handle.
E.g. 'auth.subauth/path;type=url' will return
('auth.subauth.path', {'type': 'url'}).
This also interprets % quoting in the non-option part.
"""
hdl, attrs = urllib.splitattr(hdl)
d = {}
if attrs:
for attr in attrs:
i = string.find(attr, '=')
if i < 0:
key, value = attr, None
else:
key, value = attr[:i], urllib.unquote(attr[i+1:])
d[string.lower(key)] = value
return urllib.unquote(hdl), d
def gotHeaders(self, headers):
""" The downloader will feeds headers via this function """
debug(str(self) + ' gotHeaders')
self.isGzipped = headers.get('content-encoding', [None])[0] == 'gzip'
# Grab the file name of the NZB via content-disposition header
keys = headers.keys()
found = None
for key in keys:
if key.lower() == 'content-disposition':
found = key
break
if found is None:
return
type, attrs = splitattr(headers[found][0])
key, val = splitvalue(attrs[0].strip())
val = val.strip().strip('"')
if val:
debug(str(self) + ' gotHeaders: found filename: %s' % val)
self.nzbFilename = val
def parse_url(url, default_port=None):
'''
Parse url in the following form:
PROTO://[USER:[:PASSWD]@]HOST[:PORT][/PATH[;ATTR][?QUERY]]
A tuple containing (proto, user, passwd, host, port, path, tag, attrs, query) is returned,
where `attrs' is a tuple containing ('attr1=value1', 'attr2=value2', ...)
'''
proto, user, passwd, host, port, path, tag, attrs, query = (None, ) * 9
try:
proto, tmp_host = urllib.splittype(url)
tmp_host, tmp_path = urllib.splithost(tmp_host)
tmp_user, tmp_host = urllib.splituser(tmp_host)
if tmp_user:
user, passwd = urllib.splitpasswd(tmp_user)
host, port = urllib.splitport(tmp_host)
port = int(port) if port else default_port
tmp_path, query = urllib.splitquery(tmp_path)
tmp_path, attrs = urllib.splitattr(tmp_path)
path, tag = urllib.splittag(tmp_path)
except Exception, err:
raise Exception('parse_db_url error - {0}'.format(str(err)))
def parse_url(url, default_port=None):
'''
Parse url in the following form:
PROTO://[USER:[:PASSWD]@]HOST[:PORT][/PATH[;ATTR][?QUERY]]
A tuple containing (proto, user, passwd, host, port, path, tag, attrs, query) is returned,
where `attrs' is a tuple containing ('attr1=value1', 'attr2=value2', ...)
'''
proto, user, passwd, host, port, path, tag, attrs, query = (None, ) * 9
try:
proto, tmp_host = urllib.splittype(url)
tmp_host, tmp_path = urllib.splithost(tmp_host)
tmp_user, tmp_host = urllib.splituser(tmp_host)
if tmp_user:
user, passwd = urllib.splitpasswd(tmp_user)
host, port = urllib.splitport(tmp_host)
port = int(port) if port else default_port
tmp_path, query = urllib.splitquery(tmp_path)
tmp_path, attrs = urllib.splitattr(tmp_path)
path, tag = urllib.splittag(tmp_path)
except Exception, err:
raise Exception('parse_db_url error - {0}'.format(str(err)))
def gotHeaders(self, headers):
""" The downloader will feeds headers via this function """
debug(str(self) + ' gotHeaders')
self.isGzipped = headers.get('content-encoding', [None])[0] == 'gzip'
# Grab the file name of the NZB via content-disposition header
keys = headers.keys()
found = None
for key in keys:
if key.lower() == 'content-disposition':
found = key
break
if found is None:
return
type, attrs = splitattr(headers[found][0])
key, val = splitvalue(attrs[0].strip())
val = val.strip().strip('"')
if val:
debug(str(self) + ' gotHeaders: found filename: %s' % val)
self.nzbFilename = val
class FTPChunkHandler(FTPHandler):
"""The code was taken from urllib2.py.
The only difference is that offsets are supported by this class
using the REST-command. Offsets are needed for chunked loading.
"""
def ftp_open(self, req):
import mimetypes
host = req.get_host()
if not host:
raise URLError('ftp error: no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitvalue(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
# EDIT START
# get REST (file offset) from headers
rest = 0
offset = req.headers.get('Offset', None)
if offset is not None and offset > 0:
rest = offset
# EDIT END
fp, retrlen = fw.retrfile(file, type, rest)
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors, msg:
raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
def retrieve(self, url, filename=None, reporthook=None, data=None):
""" Retrieves data from the given url and returns a tuple of filename and headers
Args:
url (str): url of the data to be retrieved
filename (str, optional): filename from the url to download
reporthook: (function, optional): function that should be called for e.g. keeping an UI updated with current state
data (, optional):
Returns:
result: (filename, headers)
See Also:
urllib.URLopener
"""
self._canceled=False
url = urllib.unwrap(urllib.toBytes(url))
if self.tempcache and url in self.tempcache:
return self.tempcache[url]
type, url1 = urllib.splittype(url)
if filename is None and (not type or type == 'file'):
try:
fp = self.open_local_file(url1)
hdrs = fp.info()
fp.close()
return urllib.url2pathname(urllib.splithost(url1)[1]), hdrs
except IOError:
pass
fp = self.open(url, data)
try:
headers = fp.info()
if filename:
tfp = open(filename, 'wb')
else:
import tempfile
garbage, path = urllib.splittype(url)
garbage, path = urllib.splithost(path or "")
path, garbage = urllib.splitquery(path or "")
path, garbage = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
(fd, filename) = tempfile.mkstemp(suffix)
self.__tempfiles.append(filename)
tfp = os.fdopen(fd, 'wb')
try:
result = filename, headers
if self.tempcache is not None:
self.tempcache[url] = result
bs = 1024 * 8
size = -1
read = 0
blocknum = 0
if "content-length" in headers:
size = int(headers["Content-Length"])
if reporthook:
reporthook(blocknum, bs, size)
while not self._canceled:
block = fp.read(bs)
if block == "":
break
read += len(block)
tfp.write(block)
blocknum += 1
if reporthook:
reporthook(blocknum, bs, size)
finally:
tfp.close()
finally:
fp.close()
# raise exception if actual size does not match content-length header
if size >= 0 and read < size:
raise urllib.ContentTooShortError("retrieval incomplete: got only %i out "
"of %i bytes" % (read, size), result)
if self._canceled and os.path.exists(filename):
os.remove(filename)
return result
def main():
"""Main routine"""
mgr = NClientsOptionParser()
(opts, args) = mgr.get_opt()
url = opts.url.replace('?', ';').replace('&', ';').replace('&', ';')
logname = opts.logname
dasquery = opts.dasquery
idx = opts.idx
limit = 1
nclients = opts.nclients
minclients = opts.minclients
debug = opts.debug
headers = {'Accept': opts.accept}
urlpath, args = urllib.splitattr(url)
repeat = opts.repeat
arr = urlpath.split('/')
if arr[0] == 'http:' or arr[0] == 'https:':
host = arr[0] + '//' + arr[2]
else:
msg = 'Provided URL="%s" does not contain http:// part' % opts.url
raise Exception(msg)
method = '/' + '/'.join(arr[3:])
params = {}
for item in args:
key, val = item.split('=')
params[key] = val
# do clean-up
for filename in os.listdir('.'):
if filename.find('.log') != -1 and filename.find(logname) != -1:
os.remove(filename)
# perform action
array = []
if nclients <= 10:
array += range(1, nclients + 1)
if 10 < nclients <= 100:
array = chain(range(1, 10),
range(10, nclients + 1, 10))
if 100 < nclients <= 1000:
array = chain(range(1, 10),
range(10, 100, 10),
range(100, nclients + 1, 100))
# allow to specify the starting nclients
array = ifilter(lambda x: x >= minclients, array)
for nclients in array:
sys.stdout.write("Run job with %s clients" % nclients)
for _ in range(repeat):
runjob(nclients, host, method, params, headers, idx, limit,
debug, logname, dasquery)
sys.stdout.write('.')
print('')
# analyze results
file_list = []
for filename in os.listdir('.'):
if filename.find('.log') != -1:
file_list.append(filename)
xxx = []
yyy = []
std = []
for ifile in natsorted(file_list):
name, _ = ifile.split('.')
# ignore non related .log files and .smf
if not logname in name or not name:
continue
xxx.append(int(name.split(logname)[-1]))
mean, std2 = avg_std(ifile)
yyy.append(mean)
std.append(std2)
try:
make_plot(xxx, yyy, std, opts.filename, title=dasquery)
except Exception as e:
print(e)
print("xxx =", xxx)
print("yyy =", yyy)
print("std =", std)
def rsync_open(self, req):
logger.debug('Opening rsync')
source, attrs = urllib.splitattr(req.get_full_url())
return self.do_rsync(source, attrs=attrs)
if not filename and (not type or type == 'file'):
try:
fp = self.open_local_file(url1)
hdrs = fp.info()
del fp
return url2pathname(urllib.splithost(url1)[1]), hdrs
except IOError, msg:
pass
fp = self.open(url)
headers = fp.info()
if not filename:
import tempfile
garbage, path = urllib.splittype(url)
garbage, path = urllib.splithost(path or "")
path, garbage = urllib.splitquery(path or "")
path, garbage = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
filename = tempfile.mktemp(suffix)
self.__tempfiles.append(filename)
result = filename, headers
if self.tempcache is not None:
self.tempcache[url] = result
tfp = open(filename, 'wb')
bs = blocksize
size = -1
blocknum = 1
if reporthook:
if headers.has_key("content-length"):
size = int(headers["Content-Length"])
stayopen = reporthook(0, bs, size, self._userObject)
if stayopen==0:
def smb_open(self, req):
global USE_NTLM, MACHINE_NAME
host = req.get_host()
if not host:
raise urllib2.URLError('SMB error: no host given')
host, port = splitport(host)
if port is None:
port = 139
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
passwd = passwd or ''
myname = MACHINE_NAME or self.generateClientMachineName()
n = NetBIOS()
names = n.queryIPForName(host)
if names:
server_name = names[0]
else:
raise urllib2.URLError(
'SMB error: Hostname does not reply back with its machine name'
)
path, attrs = splitattr(req.get_selector())
if path.startswith('/'):
path = path[1:]
dirs = path.split('/')
dirs = map(unquote, dirs)
service, path = dirs[0], '/'.join(dirs[1:])
try:
conn = SMBConnection(user,
passwd,
myname,
server_name,
use_ntlm_v2=USE_NTLM)
conn.connect(host, port)
if req.has_data():
data_fp = req.get_data()
filelen = conn.storeFile(service, path, data_fp)
headers = "Content-length: 0\n"
fp = StringIO("")
else:
fp = self.createTempFile()
file_attrs, retrlen = conn.retrieveFile(service, path, fp)
fp.seek(0)
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except Exception, ex:
raise urllib2.URLError, ('smb error: %s' % ex), sys.exc_info()[2]
class FTPRangeHandler(urllib2.FTPHandler):
def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError, ('ftp error', 'no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise urllib2.URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
# -- range support modifications start here
rest = None
range_tup = range_header_to_tuple(req.headers.get('Range', None))
assert range_tup != ()
if range_tup:
(fb, lb) = range_tup
if fb > 0:
rest = fb
# -- range support modifications end here
fp, retrlen = fw.retrfile(file, type, rest)
# -- range support modifications start here
if range_tup:
(fb, lb) = range_tup
if lb == '':
if retrlen is None or retrlen == 0:
raise RangeError('Requested Range Not Satisfiable due to unobtainable file length.')
lb = retrlen
retrlen = lb - fb
if retrlen < 0:
# beginning of range is larger than file
raise RangeError('Requested Range Not Satisfiable')
else:
retrlen = lb - fb
fp = RangeableFileObject(fp, (0, retrlen))
# -- range support modifications end here
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-Type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors, msg:
raise IOError, ('ftp error', msg), sys.exc_info()[2]
def main():
"""Main routine"""
mgr = NClientsOptionParser()
(opts, args) = mgr.get_opt()
url = opts.url.replace('?', ';').replace('&', ';').replace('&', ';')
logname = opts.logname
dasquery = opts.dasquery
idx = opts.idx
limit = 1
nclients = opts.nclients
debug = opts.debug
headers = {'Accept': opts.accept}
urlpath, args = urllib.splitattr(url)
arr = urlpath.split('/')
if arr[0] == 'http:' or arr[0] == 'https:':
host = arr[0] + '//' + arr[2]
else:
msg = 'Provided URL="%s" does not contain http:// part' % opts.url
raise Exception(msg)
method = '/' + '/'.join(arr[3:])
params = {}
for item in args:
key, val = item.split('=')
params[key] = val
# do clean-up
for filename in os.listdir('.'):
if filename.find('.log') != -1 and filename.find(logname) != -1:
os.remove(filename)
# perform action
array = []
if nclients <= 10:
array += range(1, nclients+1)
if nclients <= 100 and nclients > 10:
array = range(1, 10)
array += range(10, nclients+1, 10)
if nclients <= 1000 and nclients > 100:
array = range(1, 10)
array += range(10, 100, 10)
array += range(100, nclients+1, 100)
for nclients in array:
print "Run job with %s clients" % nclients
runjob(nclients, host, method, params, headers, idx, limit,
debug, logname, dasquery)
# analyze results
file_list = []
for filename in os.listdir('.'):
if filename.find('.log') != -1:
file_list.append(filename)
xxx = []
yyy = []
std = []
for file in natsorted(file_list):
name, _ = file.split('.')
xxx.append(int(name.split(logname)[-1]))
mean, std2 = avg_std(file)
yyy.append(mean)
std.append(std2)
try:
make_plot(xxx, yyy, std, opts.pdf)
except:
print "xxx =", xxx
print "yyy =", yyy
print "std =", std
"""An extensible library for opening URLs using a variety of protocols
def smb_open(self, req):
global USE_NTLM, MACHINE_NAME
host = req.get_host()
if not host:
raise urllib2.URLError('SMB error: no host given')
host, port = splitport(host)
if port is None:
port = 139
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
domain = ''
if ';' in user:
domain, user = user.split(';', 1)
passwd = passwd or ''
myname = MACHINE_NAME or self.generateClientMachineName()
n = NetBIOS()
names = n.queryIPForName(host)
if names:
server_name = names[0]
else:
raise urllib2.URLError('SMB error: Hostname does not reply back with its machine name')
path, attrs = splitattr(req.get_selector())
if path.startswith('/'):
path = path[1:]
dirs = path.split('/')
dirs = map(unquote, dirs)
service, path = dirs[0], '/'.join(dirs[1:])
try:
conn = SMBConnection(user, passwd, myname, server_name, domain=domain, use_ntlm_v2 = USE_NTLM)
conn.connect(host, port)
if req.has_data():
data_fp = req.get_data()
filelen = conn.storeFile(service, path, data_fp)
headers = "Content-length: 0\n"
fp = StringIO("")
else:
fp = self.createTempFile()
file_attrs, retrlen = conn.retrieveFile(service, path, fp)
fp.seek(0)
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except Exception, ex:
raise urllib2.URLError, ('smb error: %s' % ex), sys.exc_info()[2]
def __init__(self, url, method, params):
Assert(method == 'GET')
netloc, path = splithost(url)
if not netloc: raise IOError, ('ftp error', 'no host given')
host, port = splitport(netloc)
user, host = splituser(host)
if user: user, passwd = splitpasswd(user)
else: passwd = None
host = socket.gethostbyname(host)
if port:
try:
port = string.atoi(port)
except string.atoi_error:
raise IOError, ('ftp error', 'bad port')
else:
port = ftplib.FTP_PORT
path, attrs = splitattr(path)
self.url = "ftp://%s%s" % (netloc, path)
dirs = string.splitfields(path, '/')
dirs, file = dirs[:-1], dirs[-1]
self.content_length = None
if not file:
self.content_type, self.content_encoding = None, None
type = 'd'
else:
self.content_type, self.content_encoding = app.guess_type(file)
if self.content_encoding:
type = 'i'
elif self.content_type and self.content_type[:5] == 'text/':
type = 'a'
elif file[-1] == '/':
type = 'd'
else:
type = 'i'
if dirs and not dirs[0]: dirs = dirs[1:]
key = (user, host, port, string.joinfields(dirs, '/'))
self.debuglevel = None
try:
if not ftpcache.has_key(key):
ftpcache[key] = []
for attr in attrs:
[attr, value] = map(string.lower, splitvalue(attr))
if attr == 'type' and value in ('a', 'i', 'd'):
type = value
elif attr == 'debug':
try:
self.debuglevel = string.atoi(value)
except string.atoi_error:
pass
candidates = ftpcache[key]
for cand in candidates:
if not cand.busy():
break
else:
cand = ftpwrapper(user, passwd,
host, port, dirs, self.debuglevel)
candidates.append(cand)
# XXX Ought to clean the cache every once in a while
self.cand = cand
self.sock, self.isdir = cand.retrfile(file, type)
self.content_length = cand.content_length
except ftplib.all_errors, msg:
raise IOError, ('ftp error', msg)
def open_ftp(self, url):
"""Use FTP protocol."""
if not isinstance(url, str):
raise IOError(('ftp error', 'proxy support for ftp protocol currently not implemented'))
import mimetypes
import mimetools
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
host, path = urllib.splithost(url)
if not host:
raise IOError(('ftp error', 'no host given'))
host, port = urllib.splitport(host)
user, host = urllib.splituser(host)
if user:
user, passwd = urllib.splitpasswd(user)
else:
passwd = None
host = urllib.unquote(host)
user = urllib.unquote(user or '')
passwd = urllib.unquote(passwd or '')
host = socket.gethostbyname(host)
if not port:
import ftplib # noqa
port = ftplib.FTP_PORT
else:
port = int(port)
path, attrs = urllib.splitattr(path)
path = urllib.unquote(path)
dirs = path.split('/')
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
if dirs and not dirs[0]:
dirs[0] = '/'
key = user, host, port, '/'.join(dirs)
# XXX thread unsafe!
if len(self.ftpcache) > urllib.MAXFTPCACHE:
# Prune the cache, rather arbitrarily
for k in self.ftpcache.keys():
if k != key:
v = self.ftpcache[k]
del self.ftpcache[k]
v.close()
try:
if not key in self.ftpcache:
self.ftpcache[key] = \
Myftpwrapper(user, passwd, host, port, dirs)
if not file:
type = 'D'
else:
type = 'I'
for attr in attrs:
attr, value = urllib.splitvalue(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
(fp, retrlen) = self.ftpcache[key].retrfile(file, type,
rest=os.environ.get("REST"))
mtype = mimetypes.guess_type("ftp:" + url)[0]
headers = ""
if mtype:
headers += "Content-Type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
headers = mimetools.Message(StringIO(headers))
return urllib.addinfourl(fp, headers, "ftp:" + url)
except urllib.ftperrors() as msg:
raise IOError(('ftp error', msg), sys.exc_info()[2])
if not filename and (not type or type == 'file'):
try:
fp = self.open_local_file(url1)
hdrs = fp.info()
del fp
return url2pathname(urllib.splithost(url1)[1]), hdrs
except IOError, msg:
pass
fp = self.open(url)
headers = fp.info()
if not filename:
import tempfile
garbage, path = urllib.splittype(url)
garbage, path = urllib.splithost(path or "")
path, garbage = urllib.splitquery(path or "")
path, garbage = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
filename = tempfile.mktemp(suffix)
self.__tempfiles.append(filename)
result = filename, headers
if self.tempcache is not None:
self.tempcache[url] = result
tfp = open(filename, 'wb')
bs = blocksize
size = -1
blocknum = 1
if reporthook:
if headers.has_key("content-length"):
size = int(headers["Content-Length"])
stayopen = reporthook(0, bs, size, self._userObject)
if stayopen == 0:
class FancyURLopener(_OriginalFancyURLopener):
def __init__(self, *args):
apply(_OriginalFancyURLopener.__init__, (self, ) + args)
self.tempcache = {}
self.__unlink = os.unlink # See cleanup()
self.__OriginalFancyURLopener = _OriginalFancyURLopener
# prefetch support
self.__prefetchcache = {}
self.__prefetchtempfiles = {}
def __del__(self):
self.__OriginalFancyURLopener.__del__(self)
del self.__OriginalFancyURLopener
def http_error_default(self, url, fp, errcode, errmsg, headers):
void = fp.read()
fp.close()
raise IOError, (errcode, 'http error: ' + errmsg, headers)
def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
# XXX The server can force infinite recursion here!
if headers.has_key('location'):
newurl = headers['location']
elif headers.has_key('uri'):
newurl = headers['uri']
else:
return
void = fp.read()
fp.close()
fp = self.open(newurl)
h = fp.info()
if not h.has_key('Content-Location') and \
not h.has_key('Content-Base'):
h.dict['content-location'] = newurl
h.headers.append('Content-Location: %s\r\n' % newurl)
return fp
def prompt_user_passwd(self, host, realm):
import windowinterface
try:
w = windowinterface.Window('passwd', grab=1)
except AttributeError:
return _OriginalFancyURLopener.prompt_user_passwd(
self, host, realm)
l = w.Label('Enter username and password for %s at %s' % (realm, host))
t1 = w.TextInput('User:'******'',
None, (self.usercb, ()),
top=l,
left=None,
right=None)
t2 = w.TextInput('Passwd:',
'',
None, (self.passcb, ()),
modifyCB=self.modifycb,
top=t1,
left=None,
right=None)
b = w.ButtonRow([('OK', (self.do_return, ())),
('Cancel', (self.cancelcb, ()))],
vertical=0,
top=t2,
left=None,
right=None,
bottom=None)
self.userw = t1
self.passwdw = t2
self.passwd = []
self.user = ''
self.password = ''
w.show()
try:
windowinterface.mainloop()
except _end_loop:
pass
w.hide()
w.close()
del self.userw, self.passwdw
return self.user, self.password
def modifycb(self, text):
if text:
if text == '\b':
if self.passwd:
del self.passwd[-1]
return ''
self.passwd.append(text)
return '*' * len(text)
def usercb(self):
self.user = self.userw.gettext()
if self.password:
self.do_return()
else:
self.passwdw.setfocus()
def passcb(self):
self.password = string.joinfields(self.passwd, '')
if self.user:
self.do_return()
else:
self.userw.setfocus()
def cancelcb(self):
self.user = self.password = None
self.do_return()
def do_return(self):
raise _end_loop
def open_local_file(self, url):
import urlparse
scheme, netloc, url, params, query, fragment = urlparse.urlparse(url)
url = urlparse.urlunparse((scheme, netloc, url, '', '', ''))
return _OriginalFancyURLopener.open_local_file(self, url)
#
# Prefetch section
#
# override retrieve for prefetch implementation
def retrieve(self, url, filename=None, reporthook=None):
# retrieve(url) returns (filename, None) for a local object
# or (tempfilename, headers) for a remote object.
url = unwrap(url)
import urlparse
scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
if not scheme or scheme == 'file':
i = string.find(path, '?')
if i > 0:
path = path[:i]
url = urlparse.urlunparse((scheme, netloc, path, '', '', ''))
if self.__prefetchcache.has_key(url):
# complete prefetch first
#print 'completing prefetch'
self.__fin_retrieve(url)
if self.__prefetchtempfiles.has_key(url):
#print 'retrieving prefetched',self.__prefetchtempfiles[url]
return self.__prefetchtempfiles[url]
return _OriginalFancyURLopener.retrieve(self, url, filename,
reporthook)
# override cleanup for prefetch implementation
def cleanup(self):
# This code sometimes runs when the rest of this module
# has already been deleted, so it can't use any globals
# or import anything.
# first close open streams
for fp, tfp in self.__prefetchcache.values():
fp.close()
tfp.close()
self.__prefetchcache = {}
# unlink temp files
for file, header in self.__prefetchtempfiles.values():
try:
self.__unlink(file)
except:
pass
self.__prefetchtempfiles = {}
# call original cleanup
self.__OriginalFancyURLopener.cleanup(self)
# open stream to url and read headers but not data yet
# see retrieve for signature
def begin_retrieve(self, url, filename=None, reporthook=None):
url = unwrap(url)
self.__clean_retrieve(url)
type, url1 = splittype(url)
if not filename and (not type or type == 'file'):
try:
fp = self.open_local_file(url1)
hdrs = fp.info()
del fp
return url2pathname(splithost(url1)[1]), hdrs
except IOError, msg:
pass
fp = self.open(url)
headers = fp.info()
if not filename:
import tempfile
garbage, path = splittype(url)
garbage, path = splithost(path or "")
path, garbage = splitquery(path or "")
path, garbage = splitattr(path or "")
suffix = os.path.splitext(path)[1]
filename = tempfile.mktemp(suffix)
self.__prefetchtempfiles[url] = filename, headers
tfp = open(filename, 'wb')
self.__prefetchcache[url] = fp, tfp
return filename, headers
class MyFancyUrlopener(urllib.FancyURLopener):
def retrieve(self, url, filename=None, reporthook=None, data=None):
"""retrieve(url) returns (filename, headers) for a local object
or (tempfilename, headers) for a remote object."""
url = urllib.unwrap(urllib.toBytes(url))
if self.tempcache and url in self.tempcache:
return self.tempcache[url]
type, url1 = urllib.splittype(url)
if filename is None and (not type or type == 'file'):
try:
fp = self.open_local_file(url1)
hdrs = fp.info()
del fp
return urllib.url2pathname(urllib.splithost(url1)[1]), hdrs
except IOError, msg:
pass
fp = self.open(url, data)
try:
headers = fp.info()
code = fp.code
if filename:
tfp = open(filename, 'wb')
else:
import tempfile
garbage, path = urllib.splittype(url)
garbage, path = urllib.splithost(path or "")
path, garbage = urllib.splitquery(path or "")
path, garbage = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
(fd, filename) = tempfile.mkstemp(suffix)
self.__tempfiles.append(filename)
tfp = os.fdopen(fd, 'wb')
try:
result = filename, headers, code
if self.tempcache is not None:
self.tempcache[url] = result
bs = 1024 * 8
size = -1
read = 0
blocknum = 0
if reporthook:
if "content-length" in headers:
size = int(headers["Content-Length"])
reporthook(blocknum, bs, size)
while 1:
block = fp.read(bs)
if block == "":
break
read += len(block)
tfp.write(block)
blocknum += 1
if reporthook:
reporthook(blocknum, bs, size)
finally:
tfp.close()
finally:
fp.close()
del fp
del tfp
# raise exception if actual size does not match content-length header
if size >= 0 and read < size:
raise urllib.ContentTooShortError(
"retrieval incomplete: got only %i out "
"of %i bytes" % (read, size), result)
return result
def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError('ftp error', 'no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error as msg:
raise urllib2.URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
if file:
type = 'I'
else:
type = 'D'
for attr in attrs:
attr, value = splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
# -- range support modifications start here
rest = None
range_tup = range_header_to_tuple(req.headers.get('Range', None))
assert range_tup != ()
if range_tup:
(fb, lb) = range_tup
if fb > 0:
rest = fb
# -- range support modifications end here
fp, retrlen = fw.retrfile(file, type, rest)
# -- range support modifications start here
if range_tup:
(fb, lb) = range_tup
if lb == '':
if retrlen is None or retrlen == 0:
raise RangeError('Requested Range Not Satisfiable due'
' to unobtainable file length.')
lb = retrlen
retrlen = lb - fb
if retrlen < 0:
# beginning of range is larger than file
raise RangeError('Requested Range Not Satisfiable')
else:
retrlen = lb - fb
fp = RangeableFileObject(fp, (0, retrlen))
# -- range support modifications end here
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-Type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
headers = email.message_from_string(headers)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors as msg:
raise IOError('ftp error', msg)
def retrieve(self, url, filename=None, reporthook=None, data=None):
# overridden method from urllib.URLopener
self._cancelDownload = False
url = urllib.unwrap(urllib.toBytes(url))
if self.tempcache and url in self.tempcache:
return self.tempcache[url]
type, url1 = urllib.splittype(url)
if filename is None and (not type or type == 'file'):
try:
fp = self.open_local_file(url1)
hdrs = fp.info()
fp.close()
return urllib.url2pathname(urllib.splithost(url1)[1]), hdrs
except IOError:
pass
fp = self.open(url, data)
try:
headers = fp.info()
if filename:
tfp = open(filename, 'wb')
else:
import tempfile
garbage, path = urllib.splittype(url)
garbage, path = urllib.splithost(path or "")
path, garbage = urllib.splitquery(path or "")
path, garbage = urllib.splitattr(path or "")
suffix = os.path.splitext(path)[1]
(fd, filename) = tempfile.mkstemp(suffix)
self.__tempfiles.append(filename)
tfp = os.fdopen(fd, 'wb')
try:
result = filename, headers
if self.tempcache is not None:
self.tempcache[url] = result
bs = 1024 * 8
size = -1
read = 0
blocknum = 0
if "content-length" in headers:
size = int(headers["Content-Length"])
if reporthook:
reporthook(blocknum, bs, size)
while not self._cancelDownload:
block = fp.read(bs)
if block == "":
break
read += len(block)
tfp.write(block)
blocknum += 1
if reporthook:
reporthook(blocknum, bs, size)
finally:
tfp.close()
finally:
fp.close()
# raise exception if actual size does not match content-length header
if size >= 0 and read < size:
raise urllib.ContentTooShortError(
"retrieval incomplete: got only %i out "
"of %i bytes" % (read, size), result)
if self._cancelDownload and os.path.exists(filename):
os.remove(filename)
self.wasCanceled = True
return result
