def test_skip_expiry(self):
expected = (b'*2\r\n$6\r\nSELECT\r\n$1\r\n0\r\n'
b'*3\r\n$3\r\nSET\r\n$20\r\nexpires_ms_precision\r\n'
b'$27\r\n2022-12-25 10:11:12.573 UTC\r\n')
buf = BytesIO()
parser = RdbParser(ProtocolCallback(buf, emit_expire=False))
parser.parse(self.dumpfile)
self.assertEquals(buf.getvalue(), expected)
def rdb_handler(rdb):
print("rdb: ", rdb)
buf = io.BytesIO()
callback = ProtocolCallback(out=buf)
parser = RdbParser(callback)
mem_fd = io.BytesIO(rdb)
parser.parse_fd(mem_fd)
print('rdb data', buf.getvalue())
redirect_cmd(buf.getvalue())
print("parse rdb end")
def test_amend_expiry(self):
expected = (b'*2\r\n$6\r\nSELECT\r\n$1\r\n0\r\n'
b'*3\r\n$3\r\nSET\r\n$20\r\nexpires_ms_precision\r\n'
b'$27\r\n2022-12-25 10:11:12.573 UTC\r\n'
b'*3\r\n$8\r\nEXPIREAT\r\n$20\r\nexpires_ms_precision\r\n'
b'$10\r\n1671965072\r\n')
buf = BytesIO()
parser = RdbParser(ProtocolCallback(buf, amend_expire=2000))
parser.parse(self.dumpfile)
self.assertEquals(buf.getvalue(), expected)
def main():
usage = """usage: %(prog)s [options] /path/to/dump.rdb
Example : %(prog)s --command json -k "user.*" /var/redis/6379/dump.rdb"""
parser = ArgumentParser(prog='rdb', usage=usage)
parser.add_argument("-c", "--command", dest="command", required=True,
help="Command to execute. Valid commands are json, diff, justkeys, justkeyvals, memory and protocol", metavar="CMD")
parser.add_argument("-f", "--file", dest="output",
help="Output file", metavar="FILE")
parser.add_argument("-n", "--db", dest="dbs", action="append",
help="Database Number. Multiple databases can be provided. If not specified, all databases will be included.")
parser.add_argument("-k", "--key", dest="keys", default=None,
help="Keys to export. This can be a regular expression")
parser.add_argument("-o", "--not-key", dest="not_keys", default=None,
help="Keys Not to export. This can be a regular expression")
parser.add_argument("-t", "--type", dest="types", action="append",
help="""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided.
If not specified, all data types will be returned""")
parser.add_argument("-b", "--bytes", dest="bytes", default=None,
help="Limit memory output to keys greater to or equal to this value (in bytes)")
parser.add_argument("-l", "--largest", dest="largest", default=None,
help="Limit memory output to only the top N keys (by size)")
parser.add_argument("-e", "--escape", dest="escape", choices=ESCAPE_CHOICES,
help="Escape strings to encoding: %s (default), %s, %s, or %s." % tuple(ESCAPE_CHOICES))
expire_group = parser.add_mutually_exclusive_group(required=False)
expire_group.add_argument("-x", "--no-expire", dest="no_expire", default=False, action='store_true',
help="With protocol command, remove expiry from all keys")
expire_group.add_argument("-a", "--amend-expire", dest="amend_expire", default=0, type=int, metavar='N',
help="With protocol command, add N seconds to key expiry time")
parser.add_argument("dump_file", nargs=1, help="RDB Dump file to process")
options = parser.parse_args()
filters = {}
if options.dbs:
filters['dbs'] = []
for x in options.dbs:
try:
filters['dbs'].append(int(x))
except ValueError:
raise Exception('Invalid database number %s' %x)
if options.keys:
filters['keys'] = options.keys
if options.not_keys:
filters['not_keys'] = options.not_keys
if options.types:
filters['types'] = []
for x in options.types:
if not x in VALID_TYPES:
raise Exception('Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES))))
else:
filters['types'].append(x)
out_file_obj = None
try:
if options.output:
out_file_obj = open(options.output, "wb")
else:
# Prefer not to depend on Python stdout implementation for writing binary.
out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb')
try:
callback = {
'diff': lambda f: DiffCallback(f, string_escape=options.escape),
'json': lambda f: JSONCallback(f, string_escape=options.escape),
'justkeys': lambda f: KeysOnlyCallback(f, string_escape=options.escape),
'justkeyvals': lambda f: KeyValsOnlyCallback(f, string_escape=options.escape),
'memory': lambda f: MemoryCallback(PrintAllKeys(f, options.bytes, options.largest),
64, string_escape=options.escape),
'protocol': lambda f: ProtocolCallback(f, string_escape=options.escape,
emit_expire=not options.no_expire,
amend_expire=options.amend_expire
)
}[options.command](out_file_obj)
except:
raise Exception('Invalid Command %s' % options.command)
if not PYTHON_LZF_INSTALLED:
eprint("WARNING: python-lzf package NOT detected. " +
"Parsing dump file will be very slow unless you install it. " +
"To install, run the following command:")
eprint("")
eprint("pip install python-lzf")
eprint("")
parser = RdbParser(callback, filters=filters)
parser.parse(options.dump_file[0])
finally:
if options.output and out_file_obj is not None:
out_file_obj.close()
def main():
usage = """usage: %prog [options] /path/to/dump.rdb
Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb"""
parser = OptionParser(usage=usage)
parser.add_option(
"-c",
"--command",
dest="command",
help=
"Command to execute. Valid commands are json, diff, justkeys, justkeyvals and protocol",
metavar="FILE")
parser.add_option("-f",
"--file",
dest="output",
help="Output file",
metavar="FILE")
parser.add_option(
"-n",
"--db",
dest="dbs",
action="append",
help=
"Database Number. Multiple databases can be provided. If not specified, all databases will be included."
)
parser.add_option("-k",
"--key",
dest="keys",
default=None,
help="Keys to export. This can be a regular expression")
parser.add_option(
"-o",
"--not-key",
dest="not_keys",
default=None,
help="Keys Not to export. This can be a regular expression")
parser.add_option(
"-t",
"--type",
dest="types",
action="append",
help=
"""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided.
If not specified, all data types will be returned""")
parser.add_option(
"-b",
"--bytes",
dest="bytes",
default=None,
help=
"Limit memory output to keys greater to or equal to this value (in bytes)"
)
parser.add_option(
"-l",
"--largest",
dest="largest",
default=None,
help="Limit memory output to only the top N keys (by size)")
parser.add_option(
"-e",
"--escape",
dest="escape",
choices=ESCAPE_CHOICES,
help="Escape strings to encoding: %s (default), %s, %s, or %s." %
tuple(ESCAPE_CHOICES))
(options, args) = parser.parse_args()
if len(args) == 0:
parser.error("Redis RDB file not specified")
dump_file = args[0]
filters = {}
if options.dbs:
filters['dbs'] = []
for x in options.dbs:
try:
filters['dbs'].append(int(x))
except ValueError:
raise Exception('Invalid database number %s' % x)
if options.keys:
filters['keys'] = options.keys
if options.not_keys:
filters['not_keys'] = options.not_keys
if options.types:
filters['types'] = []
for x in options.types:
if not x in VALID_TYPES:
raise Exception(
'Invalid type provided - %s. Expected one of %s' %
(x, (", ".join(VALID_TYPES))))
else:
filters['types'].append(x)
out_file_obj = None
try:
if options.output:
out_file_obj = open(options.output, "wb")
else:
# Prefer not to depend on Python stdout implementation for writing binary.
out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb')
try:
callback = {
'diff':
lambda f: DiffCallback(f, string_escape=options.escape),
'json':
lambda f: JSONCallback(f, string_escape=options.escape),
'justkeys':
lambda f: KeysOnlyCallback(f, string_escape=options.escape),
'justkeyvals':
lambda f: KeyValsOnlyCallback(f, string_escape=options.escape),
'memory':
lambda f: MemoryCallback(PrintAllKeys(f, options.bytes, options
.largest),
64,
string_escape=options.escape),
'protocol':
lambda f: ProtocolCallback(f, string_escape=options.escape)
}[options.command](out_file_obj)
except:
raise Exception('Invalid Command %s' % options.command)
parser = RdbParser(callback, filters=filters)
parser.parse(dump_file)
finally:
if options.output and out_file_obj is not None:
out_file_obj.close()
def main():
usage = """usage: %prog [options] /path/to/dump.rdb
Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb"""
parser = OptionParser(usage=usage)
parser.add_option(
"-c",
"--command",
dest="command",
help="Command to execute. Valid commands are json, diff, and protocol",
metavar="FILE")
parser.add_option("-f",
"--file",
dest="output",
help="Output file",
metavar="FILE")
parser.add_option(
"-n",
"--db",
dest="dbs",
action="append",
help=
"Database Number. Multiple databases can be provided. If not specified, all databases will be included."
)
parser.add_option("-k",
"--key",
dest="keys",
default=None,
help="Keys to export. This can be a regular expression")
parser.add_option(
"-t",
"--type",
dest="types",
action="append",
help=
"""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided.
If not specified, all data types will be returned""")
(options, args) = parser.parse_args()
if len(args) == 0:
parser.error("Redis RDB file not specified")
dump_file = args[0]
filters = {}
if options.dbs:
filters['dbs'] = []
for x in options.dbs:
try:
filters['dbs'].append(int(x))
except ValueError:
raise Exception('Invalid database number %s' % x)
if options.keys:
filters['keys'] = options.keys
if options.types:
filters['types'] = []
for x in options.types:
if not x in VALID_TYPES:
raise Exception(
'Invalid type provided - %s. Expected one of %s' %
(x, (", ".join(VALID_TYPES))))
else:
filters['types'].append(x)
# TODO : Fix this ugly if-else code
if options.output:
with open(options.output, "wb") as f:
if 'diff' == options.command:
callback = DiffCallback(f)
elif 'json' == options.command:
callback = JSONCallback(f)
elif 'memory' == options.command:
reporter = PrintAllKeys(f)
callback = MemoryCallback(reporter, 64)
elif 'protocol' == options.command:
callback = ProtocolCallback(f)
else:
raise Exception('Invalid Command %s' % options.command)
parser = RdbParser(callback)
parser.parse(dump_file)
else:
if 'diff' == options.command:
callback = DiffCallback(sys.stdout)
elif 'json' == options.command:
callback = JSONCallback(sys.stdout)
elif 'memory' == options.command:
reporter = PrintAllKeys(sys.stdout)
callback = MemoryCallback(reporter, 64)
elif 'protocol' == options.command:
callback = ProtocolCallback(sys.stdout)
else:
raise Exception('Invalid Command %s' % options.command)
parser = RdbParser(callback, filters=filters)
parser.parse(dump_file)