def __call__(self, env, start_response):
Config = ConfigParser()
Config.read(configfile)
params = {"host": "", "user": "", "database": "", "port": ""}
for param in params:
if not Config.has_option("BlobStore", param):
print "Malformed configuration file: mission option %s in section %s" % (param, "BlobStore")
sys.exit(1)
params[param] = Config.get("BlobStore", param)
req = Request(env)
resp = Response(status=200, content_type="text/plain")
#engine = create_engine("mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8&use_unicode=0" %
# (params["user"], secret.BlobSecret, params["host"], params["port"], params["database"]), pool_recycle=3600)
Base = declarative_base(bind=engine)
local.Session = []
local.FileEntry = []
Session.append(sessionmaker(bind=engine))
FileEntry.append(makeFileEntry(Base))
if req.path.startswith('/fx'):
if req.method == "GET":
resp.status_code, filename, resp.content_type, resp.response = self.__download(req)
if resp.content_type == "application/octet-stream":
resp.headers["Content-Disposition"] = "attachment; filename=%s" % filename
return resp(env, start_response)
elif req.method == "POST":
resp.content_type="text/plain"
resp.response = self.__upload(req)
return resp(env, start_response)
else:
resp.status_code = 404
resp.content_type="text/plain"
resp.response = ""
return resp(env, start_response)
def __call__(self, env, start_response):
req = Request(env)
Config = ConfigParser()
Config.read(configfile)
params = {"host": "", "user": "", "database": "", "port": ""}
for param in params:
if not Config.has_option("MySQL", param):
print "Malformed configuration file: mission option %s in section MySQL" % (param)
sys.exit(1)
params[param] = Config.get("MySQL", param)
#engine = create_engine("mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8&use_unicode=0" %
# (params["user"], secret.MySQL, params["host"], params["port"], params["database"]), pool_recycle=3600)
Base = declarative_base(bind=engine)
local.Session = []
local.User = []
local.MethodEmail = []
local.Session.append(sessionmaker(bind=engine))
local.User.append(makeUser(Base))
local.MethodEmail.append(makeMethodEmail(Base))
resp = Response(status=200)
if req.path == '/register':
resp.content_type = "text/html"
resp.response = self.__register(req)
return resp(env, start_response)
elif req.path == '/regemail1':
resp.content_type = "text/html"
resp.response = self.__addEmailAuth1(req)
return resp(env, start_response)
elif req.path == '/regemail2':
resp.content_type = "text/html"
resp.response = self.__addEmailAuth2(req)
return resp(env, start_response)
elif req.path == '/regemail3':
resp.content_type = "text/html"
resp.response = self.__addEmailAuth3(req)
return resp(env, start_response)
elif req.path == '/regemailkill':
resp.content_type = "text/html"
resp.response = self.__removeEmailAuth(req)
return resp(env, start_response)
elif req.path == '/regemaillist':
resp.content_type = "text/plain"
resp.response = self.__listEmailAuth(req)
return resp(env, start_response)
elif req.path == '/auth' and req.values.has_key("email") and req.values.has_key("password"):
resp.content_type = "text/plain"
resp.response = [self.__authenticateByEmail(req)]
return resp(env, start_response)
elif req.path == '/auth' and req.values.has_key("uid"):
resp.content_type = "text/plain"
uid = self.__uid(req)
if uid == None:
uid = ""
resp.response = [uid]
return resp(env, start_response)
else:
resp.status_code = 404
resp.response = [""]
return resp(env, start_response)
def test_wrapper_internals():
"""Test internals of the wrappers"""
from werkzeug import Request
req = Request.from_values(data={'foo': 'bar'}, method='POST')
req._load_form_data()
assert req.form.to_dict() == {'foo': 'bar'}
# second call does not break
req._load_form_data()
assert req.form.to_dict() == {'foo': 'bar'}
# check reprs
assert repr(req) == "<Request 'http://localhost/' [POST]>"
resp = Response()
assert repr(resp) == '<Response 0 bytes [200 OK]>'
resp.data = 'Hello World!'
assert repr(resp) == '<Response 12 bytes [200 OK]>'
resp.response = iter(['Test'])
assert repr(resp) == '<Response streamed [200 OK]>'
# unicode data does not set content length
response = Response([u'Hällo Wörld'])
headers = response.get_wsgi_headers(create_environ())
assert 'Content-Length' not in headers
response = Response(['Hällo Wörld'])
headers = response.get_wsgi_headers(create_environ())
assert 'Content-Length' in headers
# check for internal warnings
print 'start'
filterwarnings('error', category=Warning)
response = Response()
environ = create_environ()
response.response = 'What the...?'
assert_raises(Warning, lambda: list(response.iter_encoded()))
assert_raises(Warning, lambda: list(response.get_app_iter(environ)))
response.direct_passthrough = True
assert_raises(Warning, lambda: list(response.iter_encoded()))
assert_raises(Warning, lambda: list(response.get_app_iter(environ)))
resetwarnings()
def __call__(self, env, start_response):
req = Request(env)
Config = ConfigParser()
Config.read(configfile)
params = {"host": "", "user": "", "database": "", "port": ""}
for param in params:
if not Config.has_option("MySQL", param):
print "Malformed configuration file: mission option %s in section MySQL" % (param)
sys.exit(1)
params[param] = Config.get("MySQL", param)
#print params
#engine = create_engine("mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8&use_unicode=0" %
# (params["user"], secret.MySQL, params["host"], params["port"], params["database"]), pool_recycle=3600)
Base = declarative_base(bind=engine)
local.Session = []
local.Package = []
local.Session.append(sessionmaker(bind=engine))
local.Package.append(makePackage(Base))
resp = Response(status=200, content_type="text/plain")
if req.path == '/pkg/upload':
resp.content_type="text/html"
resp.response = self.__upload(req)
return resp(env, start_response)
elif req.path == '/pkg/sdk':
resp.content_disposition="application/force-download"
resp.content_type="application/python"
resp.headers.add("Content-Disposition", "attachment; filename=appcfg.py")
f = open("appcfg.py").read().replace("__REPO_UPLOAD__", '"%s"' % (req.host_url + 'pkg/upload'))
resp.headers.add("Content-Length", str(len(f)))
resp.response = [f]
return resp(env, start_response)
elif req.path.startswith('/pkg'):
resp.status_code, resp.content_encoding, resp.response = self.__download(req)
return resp(env, start_response)
else:
resp.status_code = 404
resp.response = [""]
return resp(env, start_response)
def __call__(self, env, start_response):
local.application = self
req = Request(env)
resp = Response(status=200)
start_response('200 OK', [('Content-Type', 'text/plain')])
if req.path == '/start':
# Have all arguments been passed?
if not (req.values.has_key("protocol") and req.values.has_key("ncli") and req.values.has_key("nsrv") and req.values.has_key("rep")):
resp.status_code = 500
resp.response = ["One of the mandatory arguments has been omitted [protocol, ncli, nsrv, rep]"]
return resp(env, start_response)
# Does protocol have a valid value?
protocol = req.values["protocol"]
if protocol not in protocols:
resp.status_code = 500
resp.response = ["Protocol specified is not implemented; must be one of [ftp, http, udt, gridftp, torrent]"]
return resp(env, start_response)
# Do these arguments have valid numeric values?
try:
ncli, nsrv, rep = int(req.values["ncli"]), int(req.values["nsrv"]), int(req.values["rep"])
except:
resp.status_code = 500
resp.response = ["Unable to parse a mandatory numeric argument"]
return resp(env, start_response)
# Do the numeric arguments make sense?
if ncli < 1 or nsrv < 1 or rep < 1:
resp.status_code = 500
resp.response = ["Go screw yourself"]
return resp(env, start_response)
if ncli > len(clients):
resp.status_code = 500
resp.response = ["We don't have that many clients available. Current maximum is %d" % len(clients)]
return resp(env, start_response)
if nsrv > len(servers):
resp.status_code = 500
resp.response = ["We don't have that many servers available. Current maximum is %d" % len(servers)]
return resp(env, start_response)
# Has the previous experiment been finished already? For that, its nclients*replication must be equal to the number of associated measurements
res = my.query("select max(id) from experiment")
last_exp_id = res.rows[0][0]
if last_exp_id:
res = my.query("select replication, ncli from experiment where id=%s", (last_exp_id,))
rep_prev, ncli_prev = res.rows[0][0], res.rows[0][1]
res = my.query("select count(*) from measurement where experiment_id=%s", (last_exp_id,))
cnt = res.rows[0][0]
if cnt < rep_prev*ncli_prev:
resp.status_code = 500
resp.response = ["Unable to comply, experiment in progress"]
return resp(env, start_response)
f = NamedTemporaryFile(delete=False)
f.write(generatehaproxycfg(nsrv))
f.close()
local.ssh = paramiko.SSHClient()
local.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
local.ssh.connect(proxy, username='******', key_filename='/home/rumith/.ssh/awskeys.pem')
local.scp = scp.SCPClient(local.ssh.get_transport())
local.scp.put(f.name, "/etc/haproxy/haproxy.cfg")
if protocol == "http":
stdin, stdout, stderr = local.ssh.exec_command('pkill haproxy; haproxy -f /etc/haproxy/haproxy.cfg')
local.ssh.close()
os.unlink(f.name)
expcount, experiment_id = my.query("insert into experiment (started, protocol, ncli, nsrv, replication) values ((select now()), %s, %s, %s, %s)", (protocol, ncli, nsrv, rep))
# Copy the required protocol download programs to the clients used and start them
for i in range(ncli):
local.ssh = paramiko.SSHClient()
local.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
local.ssh.connect(clients[i], username='******', key_filename='/home/rumith/.ssh/id_rsa')
local.scp = scp.SCPClient(local.ssh.get_transport())
local.scp.put("protocol-%s.sh" % protocol, "/home/protobench")
if protocol == "http":
stdin, stdout, stderr = local.ssh.exec_command('cd /home/protobench && ./protocol-%s.sh %s %s 1 > /dev/null 2>&1 &' % (protocol, proxy, experiment_id))
local.ssh.close()
resp.response = ["Experiment %d x %d [%s] successfully started; replication %d" % (nsrv, ncli, protocol, rep) ]
return resp(env, start_response)
elif req.path == '/done':
# Did the client pass all the required arguments?
if not (req.values.has_key("expid") and req.values.has_key("runid") and req.values.has_key("bw")):
resp.status_code = 500
resp.response = ["One of the mandatory arguments has been omitted [expid, runid, bw]"]
return resp(env, start_response)
# Do they have legal values?
try:
expid, runid, bandwidth = int(req.values["expid"]), int(req.values["runid"]), int(req.values["bw"])
except:
resp.status_code = 500
resp.response = ["Unable to parse mandatory arguments"]
return resp(env, start_response)
# Is there really such an experiment to commit to?
res = my.query("select count(*) from experiment where id = %s", (req.values["expid"],))
expcnt = res.rows[0][0]
if expcnt == 0:
resp.status_code = 500
resp.response = ["No such experiment"]
return resp(env, start_response)
# Is this run being reported for the first time from this host?
res = my.query("select count(*) from measurement where experiment_id = %s and run_id = %s and client_host = %s", (expid, runid, req.remote_addr))
runcheck = res.rows[0][0]
if runcheck > 0:
resp.status_code = 500
resp.response = ["Such a run already exists"]
return resp(env, start_response)
# Does bandwidth have a valid value?
if bandwidth < 0:
resp.status_code = 500
resp.response = ["Bad bandwidth value"]
return resp(env, start_response)
# Commit the measurement
my.query("insert into measurement (experiment_id, run_id, bandwidth, client_host) values (%s, %s, %s, %s)", (expid, runid, bandwidth, req.remote_addr))
# Do we have enough replication for this host/experiment pair, or should we launch the task again?
res = my.query("select replication, protocol from experiment where id=%s", (expid,))
replication, protocol = res.rows[0][0], res.rows[0][1]
res = my.query("select count(*) from measurement where experiment_id=%s and client_host=%s", (expid, req.remote_addr))
runcnt = res.rows[0][0]
if replication > runcnt:
local.ssh = paramiko.SSHClient()
local.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
local.ssh.connect(req.remote_addr, username='******', key_filename='/home/rumith/.ssh/id_rsa')
if protocol == "http":
stdin, stdout, stderr = local.ssh.exec_command('cd /home/protobench && ./protocol-%s.sh %s %s %s > /dev/null 2>&1 &' % (protocol, proxy, expid, (runid + 1)))
local.ssh.close()
else:
# If we have enough replication for this one, maybe we have everything from other clients, too, and can finish the experiment?
res = my.query("select replication, ncli from experiment where id=%s", (expid,))
rep, ncli = res.rows[0][0], res.rows[0][1]
res = my.query("select count(*) from measurement where experiment_id=%s", (expid,))
cnt = res.rows[0][0]
if cnt > 0 and cnt == rep*ncli:
bw = []
res = my.query("select bandwidth from measurement where experiment_id=%s", (expid,))
for r in res.rows:
bw.append(r[0])
bsum = float(sum(bw)) / rep
sigma = 0
for i in range(rep):
delta = bw[i] - bsum
sigma += delta**2
bw, mse = bsum, sqrt(sigma/rep)
my.query("update experiment set ended=(select now()), bandwidth=%s, mse=%s where id=%s", (bw, mse, expid))
return resp(env, start_response)
elif req.path == '/status':
return resp(env, start_response)
else:
resp.status_code = 404
resp.response = ["Invalid request"]
return resp(env, start_response)
