def run_once(name, command, statements_path, results_path):
global normalize
server = subprocess.Popen(command + " backend=" + name, shell = True)
client = None
for i in xrange(10):
try:
client = VoltQueryClient("localhost", 21212)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error:
time.sleep(1)
if client == None:
return -1
statements_file = open(statements_path, "rb")
results_file = open(results_path, "wb")
while True:
try:
statement = cPickle.load(statements_file)
except EOFError:
break
try:
client.onecmd("adhoc " + statement["SQL"])
except:
print >> sys.stderr, "Error occurred while executing '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
# Should kill the server now
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
tables = None
if client.response.tables != None:
tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
cPickle.dump({"Status": client.response.status,
"Info": client.response.statusString,
"Result": tables,
"Exception": str(client.response.exception)},
results_file)
results_file.close()
statements_file.close()
client.onecmd("shutdown")
server.communicate()
return server.returncode
def run_once(name, command, statements):
global normalize
server = subprocess.Popen(command + " backend=" + name, shell = True)
client = None
for i in xrange(10):
try:
client = VoltQueryClient("localhost", 21212)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error:
time.sleep(1)
if client == None:
return -1
for statement in statements:
try:
client.onecmd("adhoc " + statement["SQL"])
except:
print >> sys.stderr, "Error occurred while executing '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
# Should kill the server now
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
if client.response.tables != None:
tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
tablestr = cPickle.dumps(tables, cPickle.HIGHEST_PROTOCOL)
else:
tablestr = cPickle.dumps(None, cPickle.HIGHEST_PROTOCOL)
statement[name] = {"Status": client.response.status,
"Info": client.response.statusString,
"Result": encodestring(tablestr)}
if client.response.exception != None:
statement[name]["Exception"] = str(client.response.exception)
client.onecmd("shutdown")
server.communicate()
return server.returncode
def getQueryClient():
host = defaultHost
port = defaultPort
client = None
for i in xrange(25):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
return client
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect python client to server"
sys.stderr.flush()
return None
return client
def getClient():
host = defaultHost
port = defaultPort
client = None
for i in xrange(10):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect/create client"
sys.stderr.flush()
exit(1)
return client
def connect_to_server():
global client
filename = None
if len(sys.argv) >= 2:
filename = sys.argv[1]
try:
if client != None:
client.close()
print "Connecting to server at ", volt_server_ip, \
" on port ", volt_server_port
client = VoltQueryClient(volt_server_ip,
volt_server_port,
volt_username,
volt_password,
dump_file=filename)
client.set_quiet(True)
except socket.error:
raise IOError("Error connecting to the server")
def getQueryClient(timeout=60):
host = defaultHost
port = defaultPort
client = None
endtime = time.time() + timeout
while (time.time() < endtime):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
return client
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect python client to server after %d seconds" % timeout
sys.stderr.flush()
return None
return client
def run_once(name, command, statements):
global normalize
server = subprocess.Popen(command + " backend=" + name, shell=True)
client = None
for i in xrange(10):
try:
client = VoltQueryClient("localhost", 21212)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error:
time.sleep(1)
if client == None:
return -1
for statement in statements:
try:
client.onecmd("adhoc " + statement["SQL"])
except:
print >> sys.stderr, "Error occurred while executing '%s': %s" % (statement["SQL"], sys.exc_info()[1])
# Should kill the server now
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell=True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, "Failed to kill the server process %d" % (server.pid)
break
if client.response.tables != None:
tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
tablestr = cPickle.dumps(tables, cPickle.HIGHEST_PROTOCOL)
else:
tablestr = cPickle.dumps(None, cPickle.HIGHEST_PROTOCOL)
statement[name] = {
"Status": client.response.status,
"Info": client.response.statusString,
"Result": encodestring(tablestr),
}
if client.response.exception != None:
statement[name]["Exception"] = str(client.response.exception)
client.onecmd("shutdown")
server.communicate()
return server.returncode
def getClient():
host = defaultHost
port = defaultPort
client = None
for i in xrange(10):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect/create client"
sys.stderr.flush()
exit(1)
return client
def getQueryClient():
host = defaultHost
port = defaultPort
client = None
for i in xrange(25):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
return client
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect python client to server"
sys.stderr.flush()
return None
return client
def getQueryClient(timeout=60):
host = defaultHost
port = defaultPort
client = None
endtime = time.time() + timeout
while (time.time() < endtime):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
return client
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect python client to server after %d seconds" % timeout
sys.stderr.flush()
return None
return client
def run_once(name, command, statements_path, results_path, submit_verbosely,
testConfigKit, precision):
print "Running \"run_once\":"
print " name: %s" % (name)
print " command: %s" % (command)
print " statements_path: %s" % (statements_path)
print " results_path: %s" % (results_path)
if precision:
print " precision: %s" % (precision)
sys.stdout.flush()
host = defaultHost
port = defaultPort
if (name == "jni"):
akey = "hostname"
if akey in testConfigKit:
host = testConfigKit["hostname"]
port = testConfigKit["hostport"]
global normalize
if (host == defaultHost):
server = subprocess.Popen(command + " backend=" + name, shell=True)
client = None
clientException = None
for i in xrange(30):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error as e:
clientException = e
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect/create client: there may be a problem with the VoltDB server or its ports:"
print >> sys.stderr, "name:", str(name)
print >> sys.stderr, "host:", str(host)
print >> sys.stderr, "port:", str(port)
print >> sys.stderr, "client (socket.error) exception:", str(
clientException)
sys.stderr.flush()
return -1
# for key in testConfigKits:
# print "999 Key = '%s', Val = '%s'" % (key, testConfigKits[key])
if (host != defaultHost):
# Flush database
client.onecmd("updatecatalog " + testConfigKit["testCatalog"] + " " +
testConfigKit["deploymentFile"])
statements_file = open(statements_path, "rb")
results_file = open(results_path, "wb")
while True:
try:
statement = cPickle.load(statements_file)
except EOFError:
break
try:
if submit_verbosely:
print "Submitting to backend " + name + " adhoc " + statement[
"SQL"]
client.onecmd("adhoc " + statement["SQL"])
except:
print >> sys.stderr, "Error occurred while executing '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
if (host == defaultHost):
# Should kill the server now
killer = subprocess.Popen("kill -9 %d" % (server.pid),
shell=True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
table = None
if client.response == None:
print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
if (host == defaultHost):
killer = subprocess.Popen("kill -9 %d" % (server.pid),
shell=True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
if client.response.tables:
### print "DEBUG: got table(s) from ", statement["SQL"] ,"."
if precision:
table = normalize(client.response.tables[0], statement["SQL"],
precision)
else:
table = normalize(client.response.tables[0], statement["SQL"])
if len(client.response.tables) > 1:
print "WARNING: ignoring extra table(s) from result of query ?", statement[
"SQL"], "?"
# else:
# print "WARNING: returned no table(s) from ?", statement["SQL"] ,"?"
cPickle.dump(
{
"Status": client.response.status,
"Info": client.response.statusString,
"Result": table,
"Exception": str(client.response.exception)
}, results_file)
results_file.close()
statements_file.close()
if (host == defaultHost):
client.onecmd("shutdown")
server.communicate()
else:
client.onecmd("disconnect")
sys.stdout.flush()
sys.stderr.flush()
if (host == defaultHost):
return server.returncode
else:
return 0
def run_once(name, command, statements_path, results_path, testConfigKit):
print "Running \"run_once\":"
print " name: %s" % (name)
print " command: %s" % (command)
print " statements_path: %s" % (statements_path)
print " results_path: %s" % (results_path)
sys.stdout.flush()
host = defaultHost
port = defaultPort
if(name == "jni"):
akey = "hostname"
if akey in testConfigKit:
host = testConfigKit["hostname"]
port = testConfigKit["hostport"]
global normalize
if(host == defaultHost):
server = subprocess.Popen(command + " backend=" + name, shell = True)
client = None
for i in xrange(10):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect/create client"
sys.stderr.flush()
return -1
# for key in testConfigKits:
# print "999 Key = '%s', Val = '%s'" % (key, testConfigKits[key])
if(host != defaultHost):
# Flush database
client.onecmd("updatecatalog " + testConfigKit["flushCatalog"] + " " + testConfigKit["deploymentFile"])
client.onecmd("updatecatalog " + testConfigKit["testCatalog"] + " " + testConfigKit["deploymentFile"])
statements_file = open(statements_path, "rb")
results_file = open(results_path, "wb")
while True:
try:
statement = cPickle.load(statements_file)
except EOFError:
break
try:
### print "VERBOSELY SPOUTING SENDING adhoc %s" % statement["SQL"]
client.onecmd("adhoc " + statement["SQL"])
except:
print >> sys.stderr, "Error occurred while executing '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
if(host == defaultHost):
# Should kill the server now
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
tables = None
if client.response == None:
print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
if(host == defaultHost):
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
if client.response.tables != None:
tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
# else:
# print "DEBUG: but I got no table(s) from ?", statement["SQL"] ,"?"
cPickle.dump({"Status": client.response.status,
"Info": client.response.statusString,
"Result": tables,
"Exception": str(client.response.exception)},
results_file)
results_file.close()
statements_file.close()
if(host == defaultHost):
client.onecmd("shutdown")
server.communicate()
else:
client.onecmd("disconnect")
sys.stdout.flush()
sys.stderr.flush()
if(host == defaultHost):
return server.returncode
else:
return 0
def run_once(name, command, statements_path, results_path,
submit_verbosely, testConfigKit, precision):
print "Running \"run_once\":"
print " name: %s" % (name)
print " command: %s" % (command)
print " statements_path: %s" % (statements_path)
print " results_path: %s" % (results_path)
if precision:
print " precision: %s" % (precision)
sys.stdout.flush()
host = defaultHost
port = defaultPort
if(name == "jni"):
akey = "hostname"
if akey in testConfigKit:
host = testConfigKit["hostname"]
port = testConfigKit["hostport"]
global normalize
if(host == defaultHost):
server = subprocess.Popen(command + " backend=" + name, shell=True)
client = None
clientException = None
for i in xrange(30):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error as e:
clientException = e
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect/create client: there may be a problem with the VoltDB server or its ports:"
print >> sys.stderr, "name:", str(name)
print >> sys.stderr, "host:", str(host)
print >> sys.stderr, "port:", str(port)
print >> sys.stderr, "client (socket.error) exception:", str(clientException)
sys.stderr.flush()
return -1
# for key in testConfigKits:
# print "999 Key = '%s', Val = '%s'" % (key, testConfigKits[key])
if(host != defaultHost):
# Flush database
client.onecmd("updatecatalog " + testConfigKit["testCatalog"] + " " + testConfigKit["deploymentFile"])
statements_file = open(statements_path, "rb")
results_file = open(results_path, "wb")
while True:
try:
statement = cPickle.load(statements_file)
except EOFError:
break
try:
if submit_verbosely:
print "Submitting to backend " + name + " adhoc " + statement["SQL"]
client.onecmd("adhoc " + statement["SQL"])
except:
print >> sys.stderr, "Error occurred while executing '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
if(host == defaultHost):
# Should kill the server now
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell=True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
table = None
if client.response == None:
print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
if(host == defaultHost):
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell=True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
if client.response.tables:
### print "DEBUG: got table(s) from ", statement["SQL"] ,"."
if precision:
table = normalize(client.response.tables[0], statement["SQL"], precision)
else:
table = normalize(client.response.tables[0], statement["SQL"])
if len(client.response.tables) > 1:
print "WARNING: ignoring extra table(s) from result of query ?", statement["SQL"] , "?"
# else:
# print "WARNING: returned no table(s) from ?", statement["SQL"] ,"?"
cPickle.dump({"Status": client.response.status,
"Info": client.response.statusString,
"Result": table,
"Exception": str(client.response.exception)},
results_file)
results_file.close()
statements_file.close()
if(host == defaultHost):
client.onecmd("shutdown")
server.communicate()
else:
client.onecmd("disconnect")
sys.stdout.flush()
sys.stderr.flush()
if(host == defaultHost):
return server.returncode
else:
return 0
def run_once(name, command, statements_path, results_path):
print "Running \"run_once\":"
print " name: %s" % (name)
print " command: %s" % (command)
print " statements_path: %s" % (statements_path)
print " results_path: %s" % (results_path)
sys.stdout.flush()
global normalize
server = subprocess.Popen(command + " backend=" + name, shell = True)
client = None
for i in xrange(10):
try:
client = VoltQueryClient("localhost", 21212)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect/create client"
sys.stderr.flush()
return -1
statements_file = open(statements_path, "rb")
results_file = open(results_path, "wb")
while True:
try:
statement = cPickle.load(statements_file)
except EOFError:
break
try:
client.onecmd("adhoc " + statement["SQL"])
except:
print >> sys.stderr, "Error occurred while executing '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
# Should kill the server now
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
tables = None
if client.response == None:
print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
if client.response.tables != None:
tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
cPickle.dump({"Status": client.response.status,
"Info": client.response.statusString,
"Result": tables,
"Exception": str(client.response.exception)},
results_file)
results_file.close()
statements_file.close()
client.onecmd("shutdown")
server.communicate()
sys.stdout.flush()
sys.stderr.flush()
return server.returncode
def run_once(name, command, statements_path, results_path, testConfigKit):
print "Running \"run_once\":"
print " name: %s" % (name)
print " command: %s" % (command)
print " statements_path: %s" % (statements_path)
print " results_path: %s" % (results_path)
sys.stdout.flush()
host = defaultHost
port = defaultPort
if (name == "jni"):
akey = "hostname"
if akey in testConfigKit:
host = testConfigKit["hostname"]
port = testConfigKit["hostport"]
global normalize
if (host == defaultHost):
server = subprocess.Popen(command + " backend=" + name, shell=True)
client = None
for i in xrange(10):
try:
client = VoltQueryClient(host, port)
client.set_quiet(True)
client.set_timeout(5.0) # 5 seconds
break
except socket.error:
time.sleep(1)
if client == None:
print >> sys.stderr, "Unable to connect/create client"
sys.stderr.flush()
return -1
# for key in testConfigKits:
# print "999 Key = '%s', Val = '%s'" % (key, testConfigKits[key])
if (host != defaultHost):
# Flush database
client.onecmd("updatecatalog " + testConfigKit["flushCatalog"] + " " +
testConfigKit["deploymentFile"])
client.onecmd("updatecatalog " + testConfigKit["testCatalog"] + " " +
testConfigKit["deploymentFile"])
statements_file = open(statements_path, "rb")
results_file = open(results_path, "wb")
while True:
try:
statement = cPickle.load(statements_file)
except EOFError:
break
try:
client.onecmd("adhoc " + statement["SQL"])
except:
print >> sys.stderr, "Error occurred while executing '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
if (host == defaultHost):
# Should kill the server now
killer = subprocess.Popen("kill -9 %d" % (server.pid),
shell=True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
tables = None
if client.response == None:
print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
(statement["SQL"], sys.exc_info()[1])
if (host == defaultHost):
killer = subprocess.Popen("kill -9 %d" % (server.pid),
shell=True)
killer.communicate()
if killer.returncode != 0:
print >> sys.stderr, \
"Failed to kill the server process %d" % (server.pid)
break
if client.response.tables != None:
tables = [
normalize(t, statement["SQL"]) for t in client.response.tables
]
cPickle.dump(
{
"Status": client.response.status,
"Info": client.response.statusString,
"Result": tables,
"Exception": str(client.response.exception)
}, results_file)
results_file.close()
statements_file.close()
if (host == defaultHost):
client.onecmd("shutdown")
server.communicate()
else:
client.onecmd("disconnect")
sys.stdout.flush()
sys.stderr.flush()
if (host == defaultHost):
return server.returncode
else:
return 0
