def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError('ftp error', 'no host given')
# XXX handle custom username & password
try:
host = socket.gethostbyname(host)
except socket.error as msg:
raise URLError(msg)
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
path, attrs = splitattr(req.get_selector())
path = unquote(path)
dirs = path.split('/')
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
user = passwd = '' # XXX
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
fp, retrlen = fw.retrfile(file, type)
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-Type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors as msg:
raise IOError('ftp error', msg).with_traceback(sys.exc_info()[2])
def smb_open(self, req):
global USE_NTLM, MACHINE_NAME
if not req.host:
raise urllib.error.URLError('SMB error: no host given')
host, port = splitport(req.host)
if port is None:
port = 139
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
domain = ''
if ';' in user:
domain, user = user.split(';', 1)
passwd = passwd or ''
myname = MACHINE_NAME or self.generateClientMachineName()
server_name, host = host.split(',') if ',' in host else [None, host]
if server_name is None:
n = NetBIOS()
names = n.queryIPForName(host)
if names:
server_name = names[0]
else:
raise urllib.error.URLError(
'SMB error: Hostname does not reply back with its machine name'
)
path, attrs = splitattr(req.selector)
if path.startswith('/'):
path = path[1:]
dirs = path.split('/')
dirs = list(map(unquote, dirs))
service, path = dirs[0], '/'.join(dirs[1:])
try:
conn = SMBConnection(user,
passwd,
myname,
server_name,
domain=domain,
use_ntlm_v2=USE_NTLM)
print('about to connect')
conn.connect(host, port)
headers = email.message.Message()
if req.data:
filelen = conn.storeFile(service, path, req.data)
headers.add_header('Content-length', '0')
fp = BytesIO(b"")
else:
fp = self.createTempFile()
file_attrs, retrlen = conn.retrieveFile(service, path, fp)
fp.seek(0)
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers.add_header('Content-type', mtype)
if retrlen is not None and retrlen >= 0:
headers.add_header('Content-length', '%d' % retrlen)
return addinfourl(fp, headers, req.get_full_url())
except Exception as ex:
raise urllib.error.URLError('smb error: %s' % ex).with_traceback(
sys.exc_info()[2])
def open_ftp(self, url):
"""Use FTP protocol."""
if not isinstance(url, str):
raise IOError(
'ftp error',
'proxy support for ftp protocol currently not implemented')
import mimetypes, mimetools
try:
from io import StringIO
except ImportError:
from io import StringIO
host, path = splithost(url)
if not host: raise IOError('ftp error', 'no host given')
host, port = splitport(host)
user, host = splituser(host)
if user: user, passwd = splitpasswd(user)
else: passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
host = socket.gethostbyname(host)
if not port:
from eventlib.green import ftplib
port = ftplib.FTP_PORT
else:
port = int(port)
path, attrs = splitattr(path)
path = unquote(path)
dirs = path.split('/')
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]: dirs = dirs[1:]
if dirs and not dirs[0]: dirs[0] = '/'
key = user, host, port, '/'.join(dirs)
# XXX thread unsafe!
if len(self.ftpcache) > MAXFTPCACHE:
# Prune the cache, rather arbitrarily
for k in list(self.ftpcache.keys()):
if k != key:
v = self.ftpcache[k]
del self.ftpcache[k]
v.close()
try:
if not key in self.ftpcache:
self.ftpcache[key] = \
ftpwrapper(user, passwd, host, port, dirs)
if not file: type = 'D'
else: type = 'I'
for attr in attrs:
attr, value = splitvalue(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
(fp, retrlen) = self.ftpcache[key].retrfile(file, type)
mtype = mimetypes.guess_type("ftp:" + url)[0]
headers = ""
if mtype:
headers += "Content-Type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
headers = mimetools.Message(StringIO(headers))
return addinfourl(fp, headers, "ftp:" + url)
except ftperrors() as msg:
raise IOError('ftp error', msg).with_traceback(sys.exc_info()[2])
def ftp_open(self, req):
"""
When ftp requests are made using this handler, this function gets
called at some point, and it in turn calls the ``connect_ftp`` method.
In this subclass's reimplementation of ``connect_ftp``, the FQDN of
the request's host is needed for looking up login credentials in the
password manager. However, by the time ``connect_ftp`` is called, that
information has been stripped away, and the host argument passed to
``connect_ftp`` contains only the host's IP address instead of the
FQDN. This reimplementation of ``ftp_open``, which is little more than
a copy-and-paste from the superclass's implementation, captures the
original host FQDN before it is replaced with the IP address and saves
it for later use.
This reimplementation also ensures that the file size appears in the
response header by querying for it directly. For some FTP servers the
original implementation should handle this (``retrlen`` should contain
the file size). However, for others this can fail silently due to the
server response not matching an anticipated regular expression.
"""
import sys
import email
import socket
from urllib.error import URLError
from urllib.parse import splitattr, splitpasswd, splitvalue
from urllib.response import addinfourl
####################################################
# COPIED FROM FTPHandler.ftp_open (PYTHON 3.6.6) #
# WITH JUST A FEW ADDITIONS #
####################################################
import ftplib
import mimetypes
host = req.host
if not host:
raise URLError('ftp error: no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
passwd = passwd or ''
############################################
# DIFFERENT FROM FTPHandler.ftp_open
# save the host FQDN for later
self.last_req_host = host
############################################
try:
host = socket.gethostbyname(host)
except OSError as msg:
raise URLError(msg)
path, attrs = splitattr(req.selector)
dirs = path.split('/')
dirs = list(map(unquote, dirs))
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitvalue(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
############################################
# DIFFERENT FROM FTPHandler.ftp_open
size = fw.ftp.size(file)
############################################
fp, retrlen = fw.retrfile(file, type)
headers = ""
mtype = mimetypes.guess_type(req.full_url)[0]
if mtype:
headers += "Content-type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-length: %d\n" % retrlen
############################################
# DIFFERENT FROM FTPHandler.ftp_open
elif size is not None and size >= 0:
headers += "Content-length: %d\n" % size
############################################
headers = email.message_from_string(headers)
return addinfourl(fp, headers, req.full_url)
except ftplib.all_errors as exp:
exc = URLError('ftp error: %r' % exp)
raise exc.with_traceback(sys.exc_info()[2])
def main():
"""Main routine"""
mgr = NClientsOptionParser()
(opts, args) = mgr.get_opt()
url = opts.url.replace('?', ';').replace('&', ';').replace('&', ';')
logname = opts.logname
dasquery = opts.dasquery
idx = opts.idx
limit = 1
nclients = opts.nclients
minclients = opts.minclients
debug = opts.debug
headers = {'Accept': opts.accept}
urlpath, args = urllib.splitattr(url)
repeat = opts.repeat
arr = urlpath.split('/')
if arr[0] == 'http:' or arr[0] == 'https:':
host = arr[0] + '//' + arr[2]
else:
msg = 'Provided URL="%s" does not contain http:// part' % opts.url
raise Exception(msg)
method = '/' + '/'.join(arr[3:])
params = {}
for item in args:
key, val = item.split('=')
params[key] = val
# do clean-up
for filename in os.listdir('.'):
if filename.find('.log') != -1 and filename.find(logname) != -1:
os.remove(filename)
# perform action
array = []
if nclients <= 10:
array += range(1, nclients + 1)
if 10 < nclients <= 100:
array = chain(range(1, 10),
range(10, nclients + 1, 10))
if 100 < nclients <= 1000:
array = chain(range(1, 10),
range(10, 100, 10),
range(100, nclients + 1, 100))
# allow to specify the starting nclients
array = ifilter(lambda x: x >= minclients, array)
for nclients in array:
sys.stdout.write("Run job with %s clients" % nclients)
for _ in range(repeat):
runjob(nclients, host, method, params, headers, idx, limit,
debug, logname, dasquery)
sys.stdout.write('.')
print('')
# analyze results
file_list = []
for filename in os.listdir('.'):
if filename.find('.log') != -1:
file_list.append(filename)
xxx = []
yyy = []
std = []
for ifile in natsorted(file_list):
name, _ = ifile.split('.')
# ignore non related .log files and .smf
if not logname in name or not name:
continue
xxx.append(int(name.split(logname)[-1]))
mean, std2 = avg_std(ifile)
yyy.append(mean)
std.append(std2)
try:
make_plot(xxx, yyy, std, opts.filename, title=dasquery)
except Exception as e:
print(e)
print("xxx =", xxx)
print("yyy =", yyy)
print("std =", std)
def smb_open(self, req):
global USE_NTLM, MACHINE_NAME
host = req.get_host()
if not host:
raise urllib.error.URLError('SMB error: no host given')
host, port = splitport(host)
if port is None:
port = 139
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = user or ''
passwd = passwd or ''
myname = MACHINE_NAME or self.generateClientMachineName()
n = NetBIOS()
names = n.queryIPForName(host)
if names:
server_name = names[0]
else:
raise urllib.error.URLError('SMB error: Hostname does not reply back with its machine name')
path, attrs = splitattr(req.get_selector())
if path.startswith('/'):
path = path[1:]
dirs = path.split('/')
dirs = list(map(unquote, dirs))
service, path = dirs[0], '/'.join(dirs[1:])
try:
conn = SMBConnection(user, passwd, myname, server_name, use_ntlm_v2 = USE_NTLM)
conn.connect(host, port)
headers = email.message.Message()
if req.has_data():
data_fp = req.get_data()
filelen = conn.storeFile(service, path, data_fp)
headers.add_header('Content-length', '0')
fp = BytesIO(b"")
else:
fp = self.createTempFile()
file_attrs, retrlen = conn.retrieveFile(service, path, fp)
fp.seek(0)
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers.add_header('Content-type', mtype)
if retrlen is not None and retrlen >= 0:
headers.add_header('Content-length', '%d' % retrlen)
return addinfourl(fp, headers, req.get_full_url())
except Exception as ex:
raise urllib.error.URLError('smb error: %s' % ex).with_traceback(sys.exc_info()[2])
def ftp_open(self, req):
host = req.get_host()
if not host:
raise IOError('ftp error', 'no host given')
host, port = splitport(host)
if port is None:
port = ftplib.FTP_PORT
else:
port = int(port)
# username/password handling
user, host = splituser(host)
if user:
user, passwd = splitpasswd(user)
else:
passwd = None
host = unquote(host)
user = unquote(user or '')
passwd = unquote(passwd or '')
try:
host = socket.gethostbyname(host)
except socket.error as msg:
raise URLError(msg)
path, attrs = splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp(user, passwd, host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
# -- range support modifications start here
rest = None
range_tup = range_header_to_tuple(req.headers.get('Range', None))
assert range_tup != ()
if range_tup:
(fb, lb) = range_tup
if fb > 0: rest = fb
# -- range support modifications end here
fp, retrlen = fw.retrfile(file, type, rest)
# -- range support modifications start here
if range_tup:
(fb, lb) = range_tup
if lb == '':
if retrlen is None or retrlen == 0:
raise RangeError(
9,
'Requested Range Not Satisfiable due to unobtainable file length.'
)
lb = retrlen
retrlen = lb - fb
if retrlen < 0:
# beginning of range is larger than file
raise RangeError(9, 'Requested Range Not Satisfiable')
else:
retrlen = lb - fb
fp = RangeableFileObject(fp, (0, retrlen))
# -- range support modifications end here
headers = ""
mtype = mimetypes.guess_type(req.get_full_url())[0]
if mtype:
headers += "Content-Type: %s\n" % mtype
if retrlen is not None and retrlen >= 0:
headers += "Content-Length: %d\n" % retrlen
sf = StringIO(headers)
headers = mimetools.Message(sf)
return addinfourl(fp, headers, req.get_full_url())
except ftplib.all_errors as msg:
raise IOError('ftp error', msg).with_traceback(sys.exc_info()[2])
def main():
"""Main routine"""
mgr = NClientsOptionParser()
(opts, args) = mgr.get_opt()
url = opts.url.replace('?', ';').replace('&', ';').replace('&', ';')
logname = opts.logname
dasquery = opts.dasquery
idx = opts.idx
limit = 1
nclients = opts.nclients
minclients = opts.minclients
debug = opts.debug
headers = {'Accept': opts.accept}
urlpath, args = urllib.splitattr(url)
repeat = opts.repeat
arr = urlpath.split('/')
if arr[0] == 'http:' or arr[0] == 'https:':
host = arr[0] + '//' + arr[2]
else:
msg = 'Provided URL="%s" does not contain http:// part' % opts.url
raise Exception(msg)
method = '/' + '/'.join(arr[3:])
params = {}
for item in args:
key, val = item.split('=')
params[key] = val
# do clean-up
for filename in os.listdir('.'):
if filename.find('.log') != -1 and filename.find(logname) != -1:
os.remove(filename)
# perform action
array = []
if nclients <= 10:
array += range(1, nclients + 1)
if 10 < nclients <= 100:
array = chain(range(1, 10), range(10, nclients + 1, 10))
if 100 < nclients <= 1000:
array = chain(range(1, 10), range(10, 100, 10),
range(100, nclients + 1, 100))
# allow to specify the starting nclients
array = ifilter(lambda x: x >= minclients, array)
for nclients in array:
sys.stdout.write("Run job with %s clients" % nclients)
for _ in range(repeat):
runjob(nclients, host, method, params, headers, idx, limit, debug,
logname, dasquery)
sys.stdout.write('.')
print('')
# analyze results
file_list = []
for filename in os.listdir('.'):
if filename.find('.log') != -1:
file_list.append(filename)
xxx = []
yyy = []
std = []
for ifile in natsorted(file_list):
name, _ = ifile.split('.')
# ignore non related .log files and .smf
if not logname in name or not name:
continue
xxx.append(int(name.split(logname)[-1]))
mean, std2 = avg_std(ifile)
yyy.append(mean)
std.append(std2)
try:
make_plot(xxx, yyy, std, opts.filename, title=dasquery)
except Exception as e:
print(e)
print("xxx =", xxx)
print("yyy =", yyy)
print("std =", std)
