Example #1
0
def snapshot(client, path, name, blocking):
    '''Perform a shutdown on a VoltDB instance.

    Requires three arguments:
    client - the voltdb client connection object
    path - the path the snapshots are saved to
    name - the prefix of the snapshot filenames
    blocking - whether to block to the voltdb instance

    '''

    proc = VoltProcedure(client, '@SnapshotSave',
                         [FastSerializer.VOLTTYPE_STRING,
                          FastSerializer.VOLTTYPE_STRING,
                          FastSerializer.VOLTTYPE_INTEGER])

    snapshot = proc.call([path, name, blocking])

    if snapshot.status != 1:
        sys.stderr.write(snapshot.statusString + '\n')

        return (-1, 'Snapshot Failed')

    snapshots = scan(client, path)
    if not name in snapshots:
        sys.stderr.write('Snapshot ' + name + ' not found in VoltDB; '
                         'likely an invalid filename was used.\n'
                         'Snapshot Failed!\n')

        return (-1, 'Snapshot Failed')

    return (0, 'Snapshot Successful')
Example #2
0
def shutdown(vclient, quiesce=True):
    '''Shutdown a VoltDB cluster.

    One required argument: client - the VoltDB connection object,
    should be the leader of the cluster.

    A second argument, quiesce, can specify whether to run a @Quiesce
    on the database prior to shutting down; defaults to true.  Use
    shutdown(client, quiesce=False) to disable.

    '''

    if quiesce:
        qproc = VoltProcedure(vclient, '@Quiesce', [])
        q = qproc.call()

        if q.status != 1:
            sys.stderr.write(q.statusString + '\n')
            sys.exit('Quiesce failed, aborting shutdown')

    sdproc = VoltProcedure(vclient, '@Shutdown', [])

    # Far as I know, there aren't any failure cases for the @Shutdown
    # proc.  Saying this is going to bite me in the ass in the future.
    sdproc.call([])
Example #3
0
def schema(client, type):
    proc = VoltProcedure(client, "@SystemCatalog", [FastSerializer.VOLTTYPE_STRING])

    vtables = proc.call(["TABLES"])
    columns = proc.call(["COLUMNS"])
    primarykeys = proc.call(["PRIMARYKEYS"])

    vtables = vtables.tables[0].tuples
    columns = columns.tables[0].tuples
    primarykeys = primarykeys.tables[0].tuples

    tables = {}

    for table in vtables:
        if table[3] == type:
            tables[table[2]] = {}
            tables[table[2]]["columns"] = []
            tables[table[2]]["primary_key"] = []

    for column in columns:
        if column[2] in tables:
            tables[column[2]]["columns"].append((column[16], column[3], column[5], column[6], column[8], column[10]))

    for pk in primarykeys:
        if pk[2] in tables:
            tables[pk[2]]["primary_key"].append((pk[4], pk[3]))

    for table in tables:
        tables[table]["columns"] = sorted(tables[table]["columns"])
        tables[table]["primary_key"] = sorted(tables[table]["primary_key"])

    return tables
Example #4
0
    def run_procedure(self, procedure, args, types):
        i = 0
        volttypes = []
        for type in types:
            if type == 'VARCHAR':
                volttypes.append(getattr(FastSerializer, 'VOLTTYPE_STRING'))
                continue
            elif type == 'TINYINT' or type == 'SMALLINT' or type == 'INTEGER' \
                    or type == 'BIGINT':
                if args[i] == "" or args[i].lower() == "null":
                    args[i] = None
                else:
                    args[i] = int(args[i])
            elif type == 'FLOAT' or type == 'MONEY':
                if args[i] == "" or args[i].lower() == "null":
                    args[i] = None
                else:
                    args[i] = float(args[i])
            elif type == 'DECIMAL' or type == 'DECIMAL_STRING':
                if args[i] == "" or args[i].lower() == "null":
                    args[i] = None
                else:
                    args[i] = Decimal(args[i])
            elif type == 'TIMESTAMP':
                args[i] = datetime.strptime(args[i], '%Y-%m-%d %H:%M:%S.%f')

            volttypes.append(getattr(FastSerializer, 'VOLTTYPE_' + type))

            i += 1

        proc = VoltProcedure(self.client, procedure, volttypes)

        return proc.call(args)
Example #5
0
def data(client, tables):
    proc = VoltProcedure(client, "@AdHoc", [FastSerializer.VOLTTYPE_STRING])

    for table in tables:
        rows = proc.call(["SELECT * FROM " + table])
        tables[table]["data"] = rows.tables[0].tuples

    return tables
Example #6
0
def get_table_row_count(table_name):
    host = random.choice(options.servers)
    pyclient = FastSerializer(host=host, port=21212)
    count = VoltProcedure(pyclient, '@AdHoc', [FastSerializer.VOLTTYPE_STRING])
    resp = count.call(['select count(*) from %s' % table_name], timeout=360)
    if resp.status != 1 or len(resp.tables[0].tuples) != 1:
        print "Unexpected response to count query from host %s: %s" % (host, resp)
        raise RuntimeError()
    __tuples = resp.tables[0].tuples[0]
    result = __tuples[0]
    print "count query returned: %s" % result
    return result
Example #7
0
def get_table_row_count(table_name):
    host = random.choice(options.servers)
    pyclient = FastSerializer(host=host, port=21212)
    count = VoltProcedure(pyclient, '@AdHoc', [FastSerializer.VOLTTYPE_STRING])
    resp = count.call(['select count(*) from %s' % table_name], timeout=360)
    if resp.status != 1 or len(resp.tables[0].tuples) != 1:
        print "Unexpected response to count query from host %s: %s" % (host, resp)
        raise RuntimeError()
    __tuples = resp.tables[0].tuples[0]
    result = __tuples[0]
    print "count query returned: %s" % result
    return result
Example #8
0
    def do_define(self, command):
        if self.fs == None:
            return
        if not command:
            return self.help_define()

        parsed = command.split()
        self.safe_print("Defining stored procedure:", parsed[0])

        if getattr(self.__class__, "do_" + parsed[0], None) != None:
            self.safe_print(parsed[0], "is already defined")

        try:
            method_name = "_".join(["stored", parsed[0]])
            proc_name = "_".join(["procedure", parsed[0]])
            code = """
                def %s(self, command):
                    self.safe_print("Executing stored procedure: %s")
                    try:
                        self.response = self.__safe_call(self.%s, self.prepare_params(self.%s, command), timeout = self.__timeout)
                        self.safe_print(self.response)
                    except SyntaxError, strerr:
                        self.safe_print(strerr)
                   """ % (method_name, parsed[0], proc_name, proc_name)
            tmp = {}
            exec code.strip() in tmp
            setattr(self.__class__, "do_" + parsed[0], tmp[method_name])

            setattr(
                self.__class__, proc_name,
                VoltProcedure(self.fs, parsed[0],
                              [self.__class__.TYPES[i] for i in parsed[1:]]))
        except KeyError, strerr:
            self.safe_print("Unsupported type", strerr)
            self.help_define()
Example #9
0
def updatelogger(client, log4j):
    '''Update the log4j configuration on a VoltDB cluster.

    Takes two arguments:
    client - VoltDB connection object
    log4j - log4j XML in a string

    '''

    proc = VoltProcedure(client,
                         '@UpdateLogging',
                         [FastSerializer.VOLTTYPE_VARBINARY])

    logger = proc.call([log4j])

    if logger.status != 1:
        sys.stderr.write(logger.statusString + '\n')
        sys.exit('log4j update failed!')
Example #10
0
def restore(client, path, name):
    '''Restores a VoltDB snapshot

    Three arguments required:
    client - VoltDB connection object
    path - VoltDB snapshot path
    name - prefix for the snapshot files

    '''

    proc = VoltProcedure(client,
                         '@SnapshotRestore',
                         [FastSerializer.VOLTTYPE_STRING,
                          FastSerializer.VOLTTYPE_STRING])

    response = proc.call([path, name])

    if response.status != 1:
        sys.stderr.write(response.statusString + '\n')
        sys.exit('Snapshot restore failed!')
Example #11
0
def scan(client, path):
    '''Lists information about snapshots in a given path.

    Probably not as robust as it should be.'''

    proc = VoltProcedure(client, '@SnapshotScan',
                         [FastSerializer.VOLTTYPE_STRING])

    snapshots = proc.call([path])

    s = {}
    for snap in snapshots.tables[0].tuples:
        s[snap[1]] = {'path': snap[0],
                      'txnid': snap[2],
                      'created': snap[3],
                      'size': snap[4],
                      'tables_required': snap[5],
                      'tables_missing': snap[6],
                      'tables_incomplete': snap[7],
                      'complete': snap[8]}

    return s
Example #12
0
def update(client, catalog, deployment, verbose=False):
    '''Update the VoltDB catalogy and deployment files.

    Arguments:
    client - VoltDB connection object
    catalog - str/byte of the new catalog
    deployment - str of the new deployment XML

    '''

    if verbose:
        print('Updating catalog')

    proc = VoltProcedure(client,
                         '@UpdateApplicationCatalog',
                         [FastSerializer.VOLTTYPE_VARBINARY,
                          FastSerializer.VOLTTYPE_STRING])

    response = proc.call([catalog, deployment])

    if response.status != 1:
        sys.stderr.write(response.statusString + '\n')
        sys.exit('UpdateApplicationCatalog failed\n')
Example #13
0
    def __initialize(self, host, port, username, password, client_ssl,
                     ssl_config_file, dump_file):
        # if supportSSL:
        self.fs = FastSerializer(host=host,
                                 port=port,
                                 username=username,
                                 password=password,
                                 ssl_config_file=ssl_config_file,
                                 dump_file_path=dump_file)
        # else:
        #     self.fs = FastSerializer(host=host, port=port, username=username, password=password, dump_file_path=dump_file)

        self.adhoc = VoltProcedure(self.fs, "@AdHoc",
                                   [FastSerializer.VOLTTYPE_STRING])

        self.stat = VoltProcedure(
            self.fs, "@Statistics",
            [FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_TINYINT])

        self.snapshotsave = VoltProcedure(self.fs, "@SnapshotSave", [
            FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_STRING,
            FastSerializer.VOLTTYPE_TINYINT
        ])
        self.snapshotsavejson = VoltProcedure(self.fs, "@SnapshotSave",
                                              [FastSerializer.VOLTTYPE_STRING])
        self.snapshotscan = VoltProcedure(self.fs, "@SnapshotScan",
                                          [FastSerializer.VOLTTYPE_STRING])
        self.snapshotdelete = VoltProcedure(
            self.fs, "@SnapshotDelete",
            [FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_STRING])
        self.snapshotrestore = VoltProcedure(
            self.fs, "@SnapshotRestore",
            [FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_STRING])
        self.snapshotstatus = VoltProcedure(self.fs, "@SnapshotStatus")

        self.systemcatalog = VoltProcedure(self.fs, "@SystemCatalog",
                                           [FastSerializer.VOLTTYPE_STRING])

        self.systeminformation = VoltProcedure(
            self.fs, "@SystemInformation", [FastSerializer.VOLTTYPE_STRING])

        self.updatecatalog = VoltProcedure(
            self.fs, "@UpdateApplicationCatalog",
            [FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_STRING])

        self.quiesce = VoltProcedure(self.fs, "@Quiesce")

        self.pause = VoltProcedure(self.fs, "@Pause")

        self.resume = VoltProcedure(self.fs, "@Resume")

        self.shutdown = VoltProcedure(self.fs, "@Shutdown")

        self.promote = VoltProcedure(self.fs, "@Promote")

        self.response = None
Example #14
0
            if i >= 1:
                SQL += ', '

        SQL += ' FROM "' + args.pgschema + '".' + table + ';'
        cur.execute(SQL)
        data = cur.fetchall()

        cols = []

        for col in tables[table]['columns']:
            if col[2] == 'VARCHAR':
                cols.append(FastSerializer.VOLTTYPE_STRING)
            else:
                cols.append(getattr(FastSerializer, 'VOLTTYPE_' + col[2]))

        proc = VoltProcedure(client, table + '.insert', cols)

        if args.verbose:
            print('Populating ' + table)

        for row in data:
            try:
                proc.call(data[data.index(row)])
            except:
                sys.stderr.write('insert failed in table ' + table +
                                 ' with data:\n')
                sys.stderr.write(data[data.index(row)])

    if args.verbose:
        print('Import finished, exiting VoltDB admin mode')