def snapshot(client, path, name, blocking):
'''Perform a shutdown on a VoltDB instance.
Requires three arguments:
client - the voltdb client connection object
path - the path the snapshots are saved to
name - the prefix of the snapshot filenames
blocking - whether to block to the voltdb instance
'''
proc = VoltProcedure(client, '@SnapshotSave',
[FastSerializer.VOLTTYPE_STRING,
FastSerializer.VOLTTYPE_STRING,
FastSerializer.VOLTTYPE_INTEGER])
snapshot = proc.call([path, name, blocking])
if snapshot.status != 1:
sys.stderr.write(snapshot.statusString + '\n')
return (-1, 'Snapshot Failed')
snapshots = scan(client, path)
if not name in snapshots:
sys.stderr.write('Snapshot ' + name + ' not found in VoltDB; '
'likely an invalid filename was used.\n'
'Snapshot Failed!\n')
return (-1, 'Snapshot Failed')
return (0, 'Snapshot Successful')
def shutdown(vclient, quiesce=True):
'''Shutdown a VoltDB cluster.
One required argument: client - the VoltDB connection object,
should be the leader of the cluster.
A second argument, quiesce, can specify whether to run a @Quiesce
on the database prior to shutting down; defaults to true. Use
shutdown(client, quiesce=False) to disable.
'''
if quiesce:
qproc = VoltProcedure(vclient, '@Quiesce', [])
q = qproc.call()
if q.status != 1:
sys.stderr.write(q.statusString + '\n')
sys.exit('Quiesce failed, aborting shutdown')
sdproc = VoltProcedure(vclient, '@Shutdown', [])
# Far as I know, there aren't any failure cases for the @Shutdown
# proc. Saying this is going to bite me in the ass in the future.
sdproc.call([])
def schema(client, type):
proc = VoltProcedure(client, "@SystemCatalog", [FastSerializer.VOLTTYPE_STRING])
vtables = proc.call(["TABLES"])
columns = proc.call(["COLUMNS"])
primarykeys = proc.call(["PRIMARYKEYS"])
vtables = vtables.tables[0].tuples
columns = columns.tables[0].tuples
primarykeys = primarykeys.tables[0].tuples
tables = {}
for table in vtables:
if table[3] == type:
tables[table[2]] = {}
tables[table[2]]["columns"] = []
tables[table[2]]["primary_key"] = []
for column in columns:
if column[2] in tables:
tables[column[2]]["columns"].append((column[16], column[3], column[5], column[6], column[8], column[10]))
for pk in primarykeys:
if pk[2] in tables:
tables[pk[2]]["primary_key"].append((pk[4], pk[3]))
for table in tables:
tables[table]["columns"] = sorted(tables[table]["columns"])
tables[table]["primary_key"] = sorted(tables[table]["primary_key"])
return tables
def run_procedure(self, procedure, args, types):
i = 0
volttypes = []
for type in types:
if type == 'VARCHAR':
volttypes.append(getattr(FastSerializer, 'VOLTTYPE_STRING'))
continue
elif type == 'TINYINT' or type == 'SMALLINT' or type == 'INTEGER' \
or type == 'BIGINT':
if args[i] == "" or args[i].lower() == "null":
args[i] = None
else:
args[i] = int(args[i])
elif type == 'FLOAT' or type == 'MONEY':
if args[i] == "" or args[i].lower() == "null":
args[i] = None
else:
args[i] = float(args[i])
elif type == 'DECIMAL' or type == 'DECIMAL_STRING':
if args[i] == "" or args[i].lower() == "null":
args[i] = None
else:
args[i] = Decimal(args[i])
elif type == 'TIMESTAMP':
args[i] = datetime.strptime(args[i], '%Y-%m-%d %H:%M:%S.%f')
volttypes.append(getattr(FastSerializer, 'VOLTTYPE_' + type))
i += 1
proc = VoltProcedure(self.client, procedure, volttypes)
return proc.call(args)
def data(client, tables):
proc = VoltProcedure(client, "@AdHoc", [FastSerializer.VOLTTYPE_STRING])
for table in tables:
rows = proc.call(["SELECT * FROM " + table])
tables[table]["data"] = rows.tables[0].tuples
return tables
def get_table_row_count(table_name):
host = random.choice(options.servers)
pyclient = FastSerializer(host=host, port=21212)
count = VoltProcedure(pyclient, '@AdHoc', [FastSerializer.VOLTTYPE_STRING])
resp = count.call(['select count(*) from %s' % table_name], timeout=360)
if resp.status != 1 or len(resp.tables[0].tuples) != 1:
print "Unexpected response to count query from host %s: %s" % (host, resp)
raise RuntimeError()
__tuples = resp.tables[0].tuples[0]
result = __tuples[0]
print "count query returned: %s" % result
return result
def get_table_row_count(table_name):
host = random.choice(options.servers)
pyclient = FastSerializer(host=host, port=21212)
count = VoltProcedure(pyclient, '@AdHoc', [FastSerializer.VOLTTYPE_STRING])
resp = count.call(['select count(*) from %s' % table_name], timeout=360)
if resp.status != 1 or len(resp.tables[0].tuples) != 1:
print "Unexpected response to count query from host %s: %s" % (host, resp)
raise RuntimeError()
__tuples = resp.tables[0].tuples[0]
result = __tuples[0]
print "count query returned: %s" % result
return result
def do_define(self, command):
if self.fs == None:
return
if not command:
return self.help_define()
parsed = command.split()
self.safe_print("Defining stored procedure:", parsed[0])
if getattr(self.__class__, "do_" + parsed[0], None) != None:
self.safe_print(parsed[0], "is already defined")
try:
method_name = "_".join(["stored", parsed[0]])
proc_name = "_".join(["procedure", parsed[0]])
code = """
def %s(self, command):
self.safe_print("Executing stored procedure: %s")
try:
self.response = self.__safe_call(self.%s, self.prepare_params(self.%s, command), timeout = self.__timeout)
self.safe_print(self.response)
except SyntaxError, strerr:
self.safe_print(strerr)
""" % (method_name, parsed[0], proc_name, proc_name)
tmp = {}
exec code.strip() in tmp
setattr(self.__class__, "do_" + parsed[0], tmp[method_name])
setattr(
self.__class__, proc_name,
VoltProcedure(self.fs, parsed[0],
[self.__class__.TYPES[i] for i in parsed[1:]]))
except KeyError, strerr:
self.safe_print("Unsupported type", strerr)
self.help_define()
def updatelogger(client, log4j):
'''Update the log4j configuration on a VoltDB cluster.
Takes two arguments:
client - VoltDB connection object
log4j - log4j XML in a string
'''
proc = VoltProcedure(client,
'@UpdateLogging',
[FastSerializer.VOLTTYPE_VARBINARY])
logger = proc.call([log4j])
if logger.status != 1:
sys.stderr.write(logger.statusString + '\n')
sys.exit('log4j update failed!')
def restore(client, path, name):
'''Restores a VoltDB snapshot
Three arguments required:
client - VoltDB connection object
path - VoltDB snapshot path
name - prefix for the snapshot files
'''
proc = VoltProcedure(client,
'@SnapshotRestore',
[FastSerializer.VOLTTYPE_STRING,
FastSerializer.VOLTTYPE_STRING])
response = proc.call([path, name])
if response.status != 1:
sys.stderr.write(response.statusString + '\n')
sys.exit('Snapshot restore failed!')
def scan(client, path):
'''Lists information about snapshots in a given path.
Probably not as robust as it should be.'''
proc = VoltProcedure(client, '@SnapshotScan',
[FastSerializer.VOLTTYPE_STRING])
snapshots = proc.call([path])
s = {}
for snap in snapshots.tables[0].tuples:
s[snap[1]] = {'path': snap[0],
'txnid': snap[2],
'created': snap[3],
'size': snap[4],
'tables_required': snap[5],
'tables_missing': snap[6],
'tables_incomplete': snap[7],
'complete': snap[8]}
return s
def update(client, catalog, deployment, verbose=False):
'''Update the VoltDB catalogy and deployment files.
Arguments:
client - VoltDB connection object
catalog - str/byte of the new catalog
deployment - str of the new deployment XML
'''
if verbose:
print('Updating catalog')
proc = VoltProcedure(client,
'@UpdateApplicationCatalog',
[FastSerializer.VOLTTYPE_VARBINARY,
FastSerializer.VOLTTYPE_STRING])
response = proc.call([catalog, deployment])
if response.status != 1:
sys.stderr.write(response.statusString + '\n')
sys.exit('UpdateApplicationCatalog failed\n')
def __initialize(self, host, port, username, password, client_ssl,
ssl_config_file, dump_file):
# if supportSSL:
self.fs = FastSerializer(host=host,
port=port,
username=username,
password=password,
ssl_config_file=ssl_config_file,
dump_file_path=dump_file)
# else:
# self.fs = FastSerializer(host=host, port=port, username=username, password=password, dump_file_path=dump_file)
self.adhoc = VoltProcedure(self.fs, "@AdHoc",
[FastSerializer.VOLTTYPE_STRING])
self.stat = VoltProcedure(
self.fs, "@Statistics",
[FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_TINYINT])
self.snapshotsave = VoltProcedure(self.fs, "@SnapshotSave", [
FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_STRING,
FastSerializer.VOLTTYPE_TINYINT
])
self.snapshotsavejson = VoltProcedure(self.fs, "@SnapshotSave",
[FastSerializer.VOLTTYPE_STRING])
self.snapshotscan = VoltProcedure(self.fs, "@SnapshotScan",
[FastSerializer.VOLTTYPE_STRING])
self.snapshotdelete = VoltProcedure(
self.fs, "@SnapshotDelete",
[FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_STRING])
self.snapshotrestore = VoltProcedure(
self.fs, "@SnapshotRestore",
[FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_STRING])
self.snapshotstatus = VoltProcedure(self.fs, "@SnapshotStatus")
self.systemcatalog = VoltProcedure(self.fs, "@SystemCatalog",
[FastSerializer.VOLTTYPE_STRING])
self.systeminformation = VoltProcedure(
self.fs, "@SystemInformation", [FastSerializer.VOLTTYPE_STRING])
self.updatecatalog = VoltProcedure(
self.fs, "@UpdateApplicationCatalog",
[FastSerializer.VOLTTYPE_STRING, FastSerializer.VOLTTYPE_STRING])
self.quiesce = VoltProcedure(self.fs, "@Quiesce")
self.pause = VoltProcedure(self.fs, "@Pause")
self.resume = VoltProcedure(self.fs, "@Resume")
self.shutdown = VoltProcedure(self.fs, "@Shutdown")
self.promote = VoltProcedure(self.fs, "@Promote")
self.response = None
if i >= 1:
SQL += ', '
SQL += ' FROM "' + args.pgschema + '".' + table + ';'
cur.execute(SQL)
data = cur.fetchall()
cols = []
for col in tables[table]['columns']:
if col[2] == 'VARCHAR':
cols.append(FastSerializer.VOLTTYPE_STRING)
else:
cols.append(getattr(FastSerializer, 'VOLTTYPE_' + col[2]))
proc = VoltProcedure(client, table + '.insert', cols)
if args.verbose:
print('Populating ' + table)
for row in data:
try:
proc.call(data[data.index(row)])
except:
sys.stderr.write('insert failed in table ' + table +
' with data:\n')
sys.stderr.write(data[data.index(row)])
if args.verbose:
print('Import finished, exiting VoltDB admin mode')