def get_csv(dump_file_name): buff = BytesIO() callback = MemoryCallback(PrintAllKeys(buff, None, None), 64) parser = RdbParser(callback) parser.parse( os.path.join(os.path.dirname(__file__), 'dumps', dump_file_name)) csv = buff.getvalue().decode() return csv
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example 1 : %prog -k "user.*" -k "friends.*" -f memoryreport.html /var/redis/6379/dump.rdb Example 2 : %prog /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option( "-k", "--key", dest="keys", action="append", help= "Keys that should be grouped together. Multiple regexes can be provided" ) (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] if not options.output: output = "redis_memory_report.html" else: output = options.output stats = StatsAggregator() callback = MemoryCallback(stats, 64) parser = RdbParser(callback) parser.parse(dump_file) stats_as_json = stats.get_json() with open(os.path.join(TEMPLATES_DIR, "report.html.template"), 'r') as t: report_template = Template(t.read()) html = report_template.substitute(REPORT_JSON=stats_as_json) with open(output, 'w') as f: f.write(html)
def memory_profiler(dump_file, bytes, largest, output_file=None): out_file_obj = None try: if output_file: out_file_obj = open(output_file, "wb") else: # Prefer not to depend on Python stdout implementation for writing binary. out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb') callback = MemoryCallback(MyPrintAllKeys(out_file_obj, bytes, largest), 64, string_escape=None) parser = RdbParser(callback, filters={}) parser.parse(dump_file) finally: if output_file and out_file_obj is not None: out_file_obj.close()
def print_memory_for_key(key, host='localhost', port=6379, db=0, password=None): redis = connect_to_redis(host, port, db, password) reporter = PrintMemoryUsage() callback = MemoryCallback(reporter, 64) parser = RdbParser(callback, filters={}) parser._key = key raw_dump = redis.execute_command('dump', key) if not raw_dump: sys.stderr.write('Key %s does not exist\n' % key) sys.exit(-1) stream = StringIO(raw_dump) data_type = read_unsigned_char(stream) parser.read_object(stream, data_type)
def print_memory_for_key(key, host='localhost', port=6379, db=0, password=None): redis = connect_to_redis(host, port, db, password) reporter = PrintMemoryUsage() callback = MemoryCallback(reporter, 64) parser = RdbParser(callback, filters={}) # DUMP command only return the key data, so we hack RdbParser to inject key name as parsed bytes. parser._key = key.encode('utf-8') raw_dump = redis.execute_command('dump', key) if not raw_dump: sys.stderr.write('Key %s does not exist\n' % key) sys.exit(-1) stream = BytesIO(raw_dump) data_type = read_unsigned_char(stream) parser.read_object(stream, data_type)
def parse_rdb_to_mysql(cls, rdb_file_path, mysql_server, mysql_table_name, min_key_length, mysql_batch_insert_rows=100, mysql_batch_sleep_seconds=0.01): redis_arch = 64 string_escape = "raw" stream_handler = MySQLCallback( mysql_server=mysql_server, mysql_table_name=mysql_table_name, mysql_batch_insert_rows=mysql_batch_insert_rows, mysql_batch_sleep_seconds=mysql_batch_sleep_seconds, min_key_length=min_key_length) stream_handler.init_env() parser = RdbParser( MemoryCallback(stream_handler, redis_arch, string_escape=string_escape)) parser.parse(rdb_file_path)
def main(): usage = """usage: %(prog)s [options] /path/to/dump.rdb Example : %(prog)s --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = ArgumentParser(prog='rdb', usage=usage) parser.add_argument("-c", "--command", dest="command", required=True, help="Command to execute. Valid commands are json, diff, justkeys, justkeyvals, memory and protocol", metavar="CMD") parser.add_argument("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_argument("-n", "--db", dest="dbs", action="append", help="Database Number. Multiple databases can be provided. If not specified, all databases will be included.") parser.add_argument("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_argument("-o", "--not-key", dest="not_keys", default=None, help="Keys Not to export. This can be a regular expression") parser.add_argument("-t", "--type", dest="types", action="append", help="""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") parser.add_argument("-b", "--bytes", dest="bytes", default=None, help="Limit memory output to keys greater to or equal to this value (in bytes)") parser.add_argument("-l", "--largest", dest="largest", default=None, help="Limit memory output to only the top N keys (by size)") parser.add_argument("-e", "--escape", dest="escape", choices=ESCAPE_CHOICES, help="Escape strings to encoding: %s (default), %s, %s, or %s." % tuple(ESCAPE_CHOICES)) expire_group = parser.add_mutually_exclusive_group(required=False) expire_group.add_argument("-x", "--no-expire", dest="no_expire", default=False, action='store_true', help="With protocol command, remove expiry from all keys") expire_group.add_argument("-a", "--amend-expire", dest="amend_expire", default=0, type=int, metavar='N', help="With protocol command, add N seconds to key expiry time") parser.add_argument("dump_file", nargs=1, help="RDB Dump file to process") options = parser.parse_args() filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' %x) if options.keys: filters['keys'] = options.keys if options.not_keys: filters['not_keys'] = options.not_keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception('Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) out_file_obj = None try: if options.output: out_file_obj = open(options.output, "wb") else: # Prefer not to depend on Python stdout implementation for writing binary. out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb') try: callback = { 'diff': lambda f: DiffCallback(f, string_escape=options.escape), 'json': lambda f: JSONCallback(f, string_escape=options.escape), 'justkeys': lambda f: KeysOnlyCallback(f, string_escape=options.escape), 'justkeyvals': lambda f: KeyValsOnlyCallback(f, string_escape=options.escape), 'memory': lambda f: MemoryCallback(PrintAllKeys(f, options.bytes, options.largest), 64, string_escape=options.escape), 'protocol': lambda f: ProtocolCallback(f, string_escape=options.escape, emit_expire=not options.no_expire, amend_expire=options.amend_expire ) }[options.command](out_file_obj) except: raise Exception('Invalid Command %s' % options.command) if not PYTHON_LZF_INSTALLED: eprint("WARNING: python-lzf package NOT detected. " + "Parsing dump file will be very slow unless you install it. " + "To install, run the following command:") eprint("") eprint("pip install python-lzf") eprint("") parser = RdbParser(callback, filters=filters) parser.parse(options.dump_file[0]) finally: if options.output and out_file_obj is not None: out_file_obj.close()
def get_stats(file_name): stats = Stats() callback = MemoryCallback(stats, 64) parser = RdbParser(callback) parser.parse(os.path.join(os.path.dirname(__file__), 'dumps', file_name)) return stats.records
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option( "-c", "--command", dest="command", help="Command to execute. Valid commands are json or diff", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option( "-n", "--db", dest="dbs", action="append", help= "Database Number. Multiple databases can be provided. If not specified, all databases will be included." ) parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option( "-t", "--type", dest="types", action="append", help= """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' % x) if options.keys: filters['keys'] = options.keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception( 'Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) destination = sys.stdout if options.output: desitnaion = open(options.output, 'wb') cmds = { 'diff': DiffCallback, 'json': JSONCallback, 'memory': lambda r: MemoryCallback(r, 64) } for key, cb in cmds.items(): if key != options.command: continue with destination as f: parser = RdbParser(cb(f), filters=filters) parser.parse(dump_file) return True raise Exception('Invalid Caommand %s' % options.output)
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option( "-c", "--command", dest="command", help= "Command to execute. Valid commands are json, diff, justkeys, justkeyvals and protocol", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option( "-n", "--db", dest="dbs", action="append", help= "Database Number. Multiple databases can be provided. If not specified, all databases will be included." ) parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option( "-o", "--not-key", dest="not_keys", default=None, help="Keys Not to export. This can be a regular expression") parser.add_option( "-t", "--type", dest="types", action="append", help= """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") parser.add_option( "-b", "--bytes", dest="bytes", default=None, help= "Limit memory output to keys greater to or equal to this value (in bytes)" ) parser.add_option( "-l", "--largest", dest="largest", default=None, help="Limit memory output to only the top N keys (by size)") parser.add_option( "-e", "--escape", dest="escape", choices=ESCAPE_CHOICES, help="Escape strings to encoding: %s (default), %s, %s, or %s." % tuple(ESCAPE_CHOICES)) (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' % x) if options.keys: filters['keys'] = options.keys if options.not_keys: filters['not_keys'] = options.not_keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception( 'Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) out_file_obj = None try: if options.output: out_file_obj = open(options.output, "wb") else: # Prefer not to depend on Python stdout implementation for writing binary. out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb') try: callback = { 'diff': lambda f: DiffCallback(f, string_escape=options.escape), 'json': lambda f: JSONCallback(f, string_escape=options.escape), 'justkeys': lambda f: KeysOnlyCallback(f, string_escape=options.escape), 'justkeyvals': lambda f: KeyValsOnlyCallback(f, string_escape=options.escape), 'memory': lambda f: MemoryCallback(PrintAllKeys(f, options.bytes, options .largest), 64, string_escape=options.escape), 'protocol': lambda f: ProtocolCallback(f, string_escape=options.escape) }[options.command](out_file_obj) except: raise Exception('Invalid Command %s' % options.command) parser = RdbParser(callback, filters=filters) parser.parse(dump_file) finally: if options.output and out_file_obj is not None: out_file_obj.close()
def main(): usage = """usage: %prog [options] /path/to/dump.rdb Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb""" parser = OptionParser(usage=usage) parser.add_option( "-c", "--command", dest="command", help="Command to execute. Valid commands are json or diff", metavar="FILE") parser.add_option("-f", "--file", dest="output", help="Output file", metavar="FILE") parser.add_option( "-n", "--db", dest="dbs", action="append", help= "Database Number. Multiple databases can be provided. If not specified, all databases will be included." ) parser.add_option("-k", "--key", dest="keys", default=None, help="Keys to export. This can be a regular expression") parser.add_option( "-t", "--type", dest="types", action="append", help= """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. If not specified, all data types will be returned""") (options, args) = parser.parse_args() if len(args) == 0: parser.error("Redis RDB file not specified") dump_file = args[0] filters = {} if options.dbs: filters['dbs'] = [] for x in options.dbs: try: filters['dbs'].append(int(x)) except ValueError: raise Exception('Invalid database number %s' % x) if options.keys: filters['keys'] = options.keys if options.types: filters['types'] = [] for x in options.types: if not x in VALID_TYPES: raise Exception( 'Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES)))) else: filters['types'].append(x) # TODO : Fix this ugly if-else code if options.output: with open(options.output, "wb") as f: if 'diff' == options.command: callback = DiffCallback(f) elif 'json' == options.command: callback = JSONCallback(f) elif 'memory' == options.command: reporter = PrintAllKeys(f) callback = MemoryCallback(reporter, 64) else: raise Exception('Invalid Command %s' % options.output) parser = RdbParser(callback) parser.parse(dump_file) else: if 'diff' == options.command: callback = DiffCallback(sys.stdout) elif 'json' == options.command: callback = JSONCallback(sys.stdout) elif 'memory' == options.command: reporter = PrintAllKeys(sys.stdout) callback = MemoryCallback(reporter, 64) else: raise Exception('Invalid Command %s' % options.output) parser = RdbParser(callback, filters=filters) parser.parse(dump_file)