Ejemplo n.º 1
0
def run_once(name, command, statements_path, results_path):
    global normalize
    server = subprocess.Popen(command + " backend=" + name, shell = True)
    client = None

    for i in xrange(10):
        try:
            client = VoltQueryClient("localhost", 21212)
            client.set_quiet(True)
            client.set_timeout(5.0) # 5 seconds
            break
        except socket.error:
            time.sleep(1)

    if client == None:
        return -1

    statements_file = open(statements_path, "rb")
    results_file = open(results_path, "wb")
    while True:
        try:
            statement = cPickle.load(statements_file)
        except EOFError:
            break

        try:
            client.onecmd("adhoc " + statement["SQL"])
        except:
            print >> sys.stderr, "Error occurred while executing '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            # Should kill the server now
            killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
            killer.communicate()
            if killer.returncode != 0:
                print >> sys.stderr, \
                    "Failed to kill the server process %d" % (server.pid)
            break
        tables = None
        if client.response.tables != None:
            tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
        cPickle.dump({"Status": client.response.status,
                      "Info": client.response.statusString,
                      "Result": tables,
                      "Exception": str(client.response.exception)},
                     results_file)
    results_file.close()
    statements_file.close()

    client.onecmd("shutdown")
    server.communicate()

    return server.returncode
Ejemplo n.º 2
0
def run_once(name, command, statements):
    global normalize
    server = subprocess.Popen(command + " backend=" + name, shell=True)
    client = None

    for i in xrange(10):
        try:
            client = VoltQueryClient("localhost", 21212)
            client.set_quiet(True)
            client.set_timeout(5.0)  # 5 seconds
            break
        except socket.error:
            time.sleep(1)

    if client == None:
        return -1

    for statement in statements:
        try:
            client.onecmd("adhoc " + statement["SQL"])
        except:
            print >> sys.stderr, "Error occurred while executing '%s': %s" % (statement["SQL"], sys.exc_info()[1])
            # Should kill the server now
            killer = subprocess.Popen("kill -9 %d" % (server.pid), shell=True)
            killer.communicate()
            if killer.returncode != 0:
                print >> sys.stderr, "Failed to kill the server process %d" % (server.pid)
            break
        if client.response.tables != None:
            tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
            tablestr = cPickle.dumps(tables, cPickle.HIGHEST_PROTOCOL)
        else:
            tablestr = cPickle.dumps(None, cPickle.HIGHEST_PROTOCOL)
        statement[name] = {
            "Status": client.response.status,
            "Info": client.response.statusString,
            "Result": encodestring(tablestr),
        }
        if client.response.exception != None:
            statement[name]["Exception"] = str(client.response.exception)

    client.onecmd("shutdown")
    server.communicate()

    return server.returncode
Ejemplo n.º 3
0
def run_once(name, command, statements):
    global normalize
    server = subprocess.Popen(command + " backend=" + name, shell = True)
    client = None

    for i in xrange(10):
        try:
            client = VoltQueryClient("localhost", 21212)
            client.set_quiet(True)
            client.set_timeout(5.0) # 5 seconds
            break
        except socket.error:
            time.sleep(1)

    if client == None:
        return -1

    for statement in statements:
        try:
            client.onecmd("adhoc " + statement["SQL"])
        except:
            print >> sys.stderr, "Error occurred while executing '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            # Should kill the server now
            killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
            killer.communicate()
            if killer.returncode != 0:
                print >> sys.stderr, \
                    "Failed to kill the server process %d" % (server.pid)
            break
        if client.response.tables != None:
            tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
            tablestr = cPickle.dumps(tables, cPickle.HIGHEST_PROTOCOL)
        else:
            tablestr = cPickle.dumps(None, cPickle.HIGHEST_PROTOCOL)
        statement[name] = {"Status": client.response.status,
                           "Info": client.response.statusString,
                           "Result": encodestring(tablestr)}
        if client.response.exception != None:
            statement[name]["Exception"] = str(client.response.exception)

    client.onecmd("shutdown")
    server.communicate()

    return server.returncode
Ejemplo n.º 4
0
def getClient():
    host = defaultHost
    port = defaultPort
    client = None
    for i in xrange(10):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0)  # 5 seconds
            break
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect/create client"
        sys.stderr.flush()
        exit(1)

    return client
Ejemplo n.º 5
0
def getQueryClient():
    host = defaultHost
    port = defaultPort
    client = None
    for i in xrange(25):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0) # 5 seconds
            return client
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect python client to server"
        sys.stderr.flush()
        return None

    return client
Ejemplo n.º 6
0
def getClient():
    host = defaultHost
    port = defaultPort
    client = None
    for i in xrange(10):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0) # 5 seconds
            break
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect/create client"
        sys.stderr.flush()
        exit(1)

    return client
Ejemplo n.º 7
0
def getQueryClient():
    host = defaultHost
    port = defaultPort
    client = None
    for i in xrange(25):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0)  # 5 seconds
            return client
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect python client to server"
        sys.stderr.flush()
        return None

    return client
Ejemplo n.º 8
0
def getQueryClient(timeout=60):
    host = defaultHost
    port = defaultPort
    client = None
    endtime = time.time() + timeout
    while (time.time() < endtime):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0)  # 5 seconds
            return client
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect python client to server after %d seconds" % timeout
        sys.stderr.flush()
        return None

    return client
Ejemplo n.º 9
0
def connect_to_server():
    global client

    filename = None
    if len(sys.argv) >= 2:
        filename = sys.argv[1]

    try:
        if client != None:
            client.close()
        print "Connecting to server at ", volt_server_ip, \
            " on port ", volt_server_port
        client = VoltQueryClient(volt_server_ip,
                                 volt_server_port,
                                 volt_username,
                                 volt_password,
                                 dump_file=filename)
        client.set_quiet(True)
    except socket.error:
        raise IOError("Error connecting to the server")
Ejemplo n.º 10
0
def getQueryClient(timeout=60):
    host = defaultHost
    port = defaultPort
    client = None
    endtime = time.time() + timeout
    while (time.time() < endtime):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0) # 5 seconds
            return client
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect python client to server after %d seconds" % timeout
        sys.stderr.flush()
        return None

    return client
Ejemplo n.º 11
0
def run_once(name, command, statements_path, results_path, submit_verbosely,
             testConfigKit, precision):

    print "Running \"run_once\":"
    print "  name: %s" % (name)
    print "  command: %s" % (command)
    print "  statements_path: %s" % (statements_path)
    print "  results_path: %s" % (results_path)
    if precision:
        print "  precision: %s" % (precision)
    sys.stdout.flush()

    host = defaultHost
    port = defaultPort
    if (name == "jni"):
        akey = "hostname"
        if akey in testConfigKit:
            host = testConfigKit["hostname"]
            port = testConfigKit["hostport"]

    global normalize
    if (host == defaultHost):
        server = subprocess.Popen(command + " backend=" + name, shell=True)

    client = None
    clientException = None
    for i in xrange(30):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0)  # 5 seconds
            break
        except socket.error as e:
            clientException = e
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect/create client: there may be a problem with the VoltDB server or its ports:"
        print >> sys.stderr, "name:", str(name)
        print >> sys.stderr, "host:", str(host)
        print >> sys.stderr, "port:", str(port)
        print >> sys.stderr, "client (socket.error) exception:", str(
            clientException)
        sys.stderr.flush()
        return -1


#    for key in testConfigKits:
#        print "999 Key = '%s', Val = '%s'" % (key, testConfigKits[key])
    if (host != defaultHost):
        # Flush database
        client.onecmd("updatecatalog " + testConfigKit["testCatalog"] + " " +
                      testConfigKit["deploymentFile"])

    statements_file = open(statements_path, "rb")
    results_file = open(results_path, "wb")
    while True:
        try:
            statement = cPickle.load(statements_file)
        except EOFError:
            break

        try:
            if submit_verbosely:
                print "Submitting to backend " + name + " adhoc " + statement[
                    "SQL"]
            client.onecmd("adhoc " + statement["SQL"])
        except:
            print >> sys.stderr, "Error occurred while executing '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            if (host == defaultHost):
                # Should kill the server now
                killer = subprocess.Popen("kill -9 %d" % (server.pid),
                                          shell=True)
                killer.communicate()
                if killer.returncode != 0:
                    print >> sys.stderr, \
                        "Failed to kill the server process %d" % (server.pid)
            break
        table = None
        if client.response == None:
            print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            if (host == defaultHost):
                killer = subprocess.Popen("kill -9 %d" % (server.pid),
                                          shell=True)
                killer.communicate()
                if killer.returncode != 0:
                    print >> sys.stderr, \
                        "Failed to kill the server process %d" % (server.pid)
            break
        if client.response.tables:
            ### print "DEBUG: got table(s) from ", statement["SQL"] ,"."
            if precision:
                table = normalize(client.response.tables[0], statement["SQL"],
                                  precision)
            else:
                table = normalize(client.response.tables[0], statement["SQL"])
            if len(client.response.tables) > 1:
                print "WARNING: ignoring extra table(s) from result of query ?", statement[
                    "SQL"], "?"
        # else:
        # print "WARNING: returned no table(s) from ?", statement["SQL"] ,"?"
        cPickle.dump(
            {
                "Status": client.response.status,
                "Info": client.response.statusString,
                "Result": table,
                "Exception": str(client.response.exception)
            }, results_file)
    results_file.close()
    statements_file.close()

    if (host == defaultHost):
        client.onecmd("shutdown")
        server.communicate()
    else:
        client.onecmd("disconnect")

    sys.stdout.flush()
    sys.stderr.flush()

    if (host == defaultHost):
        return server.returncode
    else:
        return 0
Ejemplo n.º 12
0
def run_once(name, command, statements_path, results_path, testConfigKit):

    print "Running \"run_once\":"
    print "  name: %s" % (name)
    print "  command: %s" % (command)
    print "  statements_path: %s" % (statements_path)
    print "  results_path: %s" % (results_path)
    sys.stdout.flush()

    host = defaultHost
    port = defaultPort
    if(name == "jni"):
        akey = "hostname"
        if akey in testConfigKit:
            host = testConfigKit["hostname"]
            port = testConfigKit["hostport"]

    global normalize
    if(host == defaultHost):
        server = subprocess.Popen(command + " backend=" + name, shell = True)

    client = None
    for i in xrange(10):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0) # 5 seconds
            break
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect/create client"
        sys.stderr.flush()
        return -1

#    for key in testConfigKits:
#        print "999 Key = '%s', Val = '%s'" % (key, testConfigKits[key])
    if(host != defaultHost):
        # Flush database
        client.onecmd("updatecatalog " + testConfigKit["flushCatalog"] + " " + testConfigKit["deploymentFile"])
        client.onecmd("updatecatalog " + testConfigKit["testCatalog"]  + " " + testConfigKit["deploymentFile"])

    statements_file = open(statements_path, "rb")
    results_file = open(results_path, "wb")
    while True:
        try:
            statement = cPickle.load(statements_file)
        except EOFError:
            break

        try:
            ### print "VERBOSELY SPOUTING SENDING adhoc %s" % statement["SQL"]
            client.onecmd("adhoc " + statement["SQL"])
        except:
            print >> sys.stderr, "Error occurred while executing '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            if(host == defaultHost):
                # Should kill the server now
                killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
                killer.communicate()
                if killer.returncode != 0:
                    print >> sys.stderr, \
                        "Failed to kill the server process %d" % (server.pid)
            break
        tables = None
        if client.response == None:
            print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            if(host == defaultHost):
                killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
                killer.communicate()
                if killer.returncode != 0:
                    print >> sys.stderr, \
                        "Failed to kill the server process %d" % (server.pid)
            break
        if client.response.tables != None:
            tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
        # else:
        #     print "DEBUG: but I got no table(s) from ?", statement["SQL"] ,"?"
        cPickle.dump({"Status": client.response.status,
                      "Info": client.response.statusString,
                      "Result": tables,
                      "Exception": str(client.response.exception)},
                     results_file)
    results_file.close()
    statements_file.close()

    if(host == defaultHost):
        client.onecmd("shutdown")
        server.communicate()
    else:
        client.onecmd("disconnect")

    sys.stdout.flush()
    sys.stderr.flush()

    if(host == defaultHost):
        return server.returncode
    else:
        return 0
Ejemplo n.º 13
0
def run_once(name, command, statements_path, results_path,
             submit_verbosely, testConfigKit, precision):

    print "Running \"run_once\":"
    print "  name: %s" % (name)
    print "  command: %s" % (command)
    print "  statements_path: %s" % (statements_path)
    print "  results_path: %s" % (results_path)
    if precision:
        print "  precision: %s" % (precision)
    sys.stdout.flush()

    host = defaultHost
    port = defaultPort
    if(name == "jni"):
        akey = "hostname"
        if akey in testConfigKit:
            host = testConfigKit["hostname"]
            port = testConfigKit["hostport"]

    global normalize
    if(host == defaultHost):
        server = subprocess.Popen(command + " backend=" + name, shell=True)

    client = None
    clientException = None
    for i in xrange(30):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0) # 5 seconds
            break
        except socket.error as e:
            clientException = e
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect/create client: there may be a problem with the VoltDB server or its ports:"
        print >> sys.stderr, "name:", str(name)
        print >> sys.stderr, "host:", str(host)
        print >> sys.stderr, "port:", str(port)
        print >> sys.stderr, "client (socket.error) exception:", str(clientException)
        sys.stderr.flush()
        return -1

#    for key in testConfigKits:
#        print "999 Key = '%s', Val = '%s'" % (key, testConfigKits[key])
    if(host != defaultHost):
        # Flush database
        client.onecmd("updatecatalog " + testConfigKit["testCatalog"] + " " + testConfigKit["deploymentFile"])

    statements_file = open(statements_path, "rb")
    results_file = open(results_path, "wb")
    while True:
        try:
            statement = cPickle.load(statements_file)
        except EOFError:
            break

        try:
            if submit_verbosely:
                print "Submitting to backend " + name + " adhoc " + statement["SQL"]
            client.onecmd("adhoc " + statement["SQL"])
        except:
            print >> sys.stderr, "Error occurred while executing '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            if(host == defaultHost):
                # Should kill the server now
                killer = subprocess.Popen("kill -9 %d" % (server.pid), shell=True)
                killer.communicate()
                if killer.returncode != 0:
                    print >> sys.stderr, \
                        "Failed to kill the server process %d" % (server.pid)
            break
        table = None
        if client.response == None:
            print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            if(host == defaultHost):
                killer = subprocess.Popen("kill -9 %d" % (server.pid), shell=True)
                killer.communicate()
                if killer.returncode != 0:
                    print >> sys.stderr, \
                        "Failed to kill the server process %d" % (server.pid)
            break
        if client.response.tables:
            ### print "DEBUG: got table(s) from ", statement["SQL"] ,"."
            if precision:
                table = normalize(client.response.tables[0], statement["SQL"], precision)
            else:
                table = normalize(client.response.tables[0], statement["SQL"])
            if len(client.response.tables) > 1:
                print "WARNING: ignoring extra table(s) from result of query ?", statement["SQL"] , "?"
        # else:
            # print "WARNING: returned no table(s) from ?", statement["SQL"] ,"?"
        cPickle.dump({"Status": client.response.status,
                      "Info": client.response.statusString,
                      "Result": table,
                      "Exception": str(client.response.exception)},
                     results_file)
    results_file.close()
    statements_file.close()

    if(host == defaultHost):
        client.onecmd("shutdown")
        server.communicate()
    else:
        client.onecmd("disconnect")

    sys.stdout.flush()
    sys.stderr.flush()

    if(host == defaultHost):
        return server.returncode
    else:
        return 0
Ejemplo n.º 14
0
def run_once(name, command, statements_path, results_path):

    print "Running \"run_once\":"
    print "  name: %s" % (name)
    print "  command: %s" % (command)
    print "  statements_path: %s" % (statements_path)
    print "  results_path: %s" % (results_path)
    sys.stdout.flush()

    global normalize
    server = subprocess.Popen(command + " backend=" + name, shell = True)
    client = None

    for i in xrange(10):
        try:
            client = VoltQueryClient("localhost", 21212)
            client.set_quiet(True)
            client.set_timeout(5.0) # 5 seconds
            break
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect/create client"
        sys.stderr.flush()
        return -1

    statements_file = open(statements_path, "rb")
    results_file = open(results_path, "wb")
    while True:
        try:
            statement = cPickle.load(statements_file)
        except EOFError:
            break

        try:
            client.onecmd("adhoc " + statement["SQL"])
        except:
            print >> sys.stderr, "Error occurred while executing '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            # Should kill the server now
            killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
            killer.communicate()
            if killer.returncode != 0:
                print >> sys.stderr, \
                    "Failed to kill the server process %d" % (server.pid)
            break
        tables = None
        if client.response == None:
            print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            killer = subprocess.Popen("kill -9 %d" % (server.pid), shell = True)
            killer.communicate()
            if killer.returncode != 0:
                print >> sys.stderr, \
                    "Failed to kill the server process %d" % (server.pid)
            break
        if client.response.tables != None:
            tables = [normalize(t, statement["SQL"]) for t in client.response.tables]
        cPickle.dump({"Status": client.response.status,
                      "Info": client.response.statusString,
                      "Result": tables,
                      "Exception": str(client.response.exception)},
                     results_file)
    results_file.close()
    statements_file.close()

    client.onecmd("shutdown")
    server.communicate()

    sys.stdout.flush()
    sys.stderr.flush()

    return server.returncode
Ejemplo n.º 15
0
def run_once(name, command, statements_path, results_path, testConfigKit):

    print "Running \"run_once\":"
    print "  name: %s" % (name)
    print "  command: %s" % (command)
    print "  statements_path: %s" % (statements_path)
    print "  results_path: %s" % (results_path)
    sys.stdout.flush()

    host = defaultHost
    port = defaultPort
    if (name == "jni"):
        akey = "hostname"
        if akey in testConfigKit:
            host = testConfigKit["hostname"]
            port = testConfigKit["hostport"]

    global normalize
    if (host == defaultHost):
        server = subprocess.Popen(command + " backend=" + name, shell=True)

    client = None
    for i in xrange(10):
        try:
            client = VoltQueryClient(host, port)
            client.set_quiet(True)
            client.set_timeout(5.0)  # 5 seconds
            break
        except socket.error:
            time.sleep(1)

    if client == None:
        print >> sys.stderr, "Unable to connect/create client"
        sys.stderr.flush()
        return -1


#    for key in testConfigKits:
#        print "999 Key = '%s', Val = '%s'" % (key, testConfigKits[key])
    if (host != defaultHost):
        # Flush database
        client.onecmd("updatecatalog " + testConfigKit["flushCatalog"] + " " +
                      testConfigKit["deploymentFile"])
        client.onecmd("updatecatalog " + testConfigKit["testCatalog"] + " " +
                      testConfigKit["deploymentFile"])

    statements_file = open(statements_path, "rb")
    results_file = open(results_path, "wb")
    while True:
        try:
            statement = cPickle.load(statements_file)
        except EOFError:
            break

        try:
            client.onecmd("adhoc " + statement["SQL"])
        except:
            print >> sys.stderr, "Error occurred while executing '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            if (host == defaultHost):
                # Should kill the server now
                killer = subprocess.Popen("kill -9 %d" % (server.pid),
                                          shell=True)
                killer.communicate()
                if killer.returncode != 0:
                    print >> sys.stderr, \
                        "Failed to kill the server process %d" % (server.pid)
            break
        tables = None
        if client.response == None:
            print >> sys.stderr, "No error, but an unexpected null client response (server crash?) from executing statement '%s': %s" % \
                (statement["SQL"], sys.exc_info()[1])
            if (host == defaultHost):
                killer = subprocess.Popen("kill -9 %d" % (server.pid),
                                          shell=True)
                killer.communicate()
                if killer.returncode != 0:
                    print >> sys.stderr, \
                        "Failed to kill the server process %d" % (server.pid)
            break
        if client.response.tables != None:
            tables = [
                normalize(t, statement["SQL"]) for t in client.response.tables
            ]
        cPickle.dump(
            {
                "Status": client.response.status,
                "Info": client.response.statusString,
                "Result": tables,
                "Exception": str(client.response.exception)
            }, results_file)
    results_file.close()
    statements_file.close()

    if (host == defaultHost):
        client.onecmd("shutdown")
        server.communicate()
    else:
        client.onecmd("disconnect")

    sys.stdout.flush()
    sys.stderr.flush()

    if (host == defaultHost):
        return server.returncode
    else:
        return 0