Beispiel #1
0
def get_csv(dump_file_name):
    buff = BytesIO()
    callback = MemoryCallback(PrintAllKeys(buff, None, None), 64)
    parser = RdbParser(callback)
    parser.parse(
        os.path.join(os.path.dirname(__file__), 'dumps', dump_file_name))
    csv = buff.getvalue().decode()
    return csv
Beispiel #2
0
def main():
    usage = """usage: %prog [options] /path/to/dump.rdb

Example 1 : %prog -k "user.*" -k "friends.*" -f memoryreport.html /var/redis/6379/dump.rdb
Example 2 : %prog /var/redis/6379/dump.rdb"""

    parser = OptionParser(usage=usage)

    parser.add_option("-f",
                      "--file",
                      dest="output",
                      help="Output file",
                      metavar="FILE")
    parser.add_option(
        "-k",
        "--key",
        dest="keys",
        action="append",
        help=
        "Keys that should be grouped together. Multiple regexes can be provided"
    )

    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.error("Redis RDB file not specified")
    dump_file = args[0]

    if not options.output:
        output = "redis_memory_report.html"
    else:
        output = options.output

    stats = StatsAggregator()
    callback = MemoryCallback(stats, 64)
    parser = RdbParser(callback)
    parser.parse(dump_file)
    stats_as_json = stats.get_json()

    with open(os.path.join(TEMPLATES_DIR, "report.html.template"), 'r') as t:
        report_template = Template(t.read())

    html = report_template.substitute(REPORT_JSON=stats_as_json)

    with open(output, 'w') as f:
        f.write(html)
Beispiel #3
0
def memory_profiler(dump_file, bytes, largest, output_file=None):
    out_file_obj = None
    try:
        if output_file:
            out_file_obj = open(output_file, "wb")
        else:
            # Prefer not to depend on Python stdout implementation for writing binary.
            out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb')

        callback = MemoryCallback(MyPrintAllKeys(out_file_obj, bytes, largest),
                                  64,
                                  string_escape=None)
        parser = RdbParser(callback, filters={})
        parser.parse(dump_file)

    finally:
        if output_file and out_file_obj is not None:
            out_file_obj.close()
Beispiel #4
0
def print_memory_for_key(key,
                         host='localhost',
                         port=6379,
                         db=0,
                         password=None):
    redis = connect_to_redis(host, port, db, password)
    reporter = PrintMemoryUsage()
    callback = MemoryCallback(reporter, 64)
    parser = RdbParser(callback, filters={})
    parser._key = key

    raw_dump = redis.execute_command('dump', key)
    if not raw_dump:
        sys.stderr.write('Key %s does not exist\n' % key)
        sys.exit(-1)

    stream = StringIO(raw_dump)
    data_type = read_unsigned_char(stream)
    parser.read_object(stream, data_type)
def print_memory_for_key(key,
                         host='localhost',
                         port=6379,
                         db=0,
                         password=None):
    redis = connect_to_redis(host, port, db, password)
    reporter = PrintMemoryUsage()
    callback = MemoryCallback(reporter, 64)
    parser = RdbParser(callback, filters={})
    #  DUMP command only return the key data, so we hack RdbParser to inject key name as parsed bytes.
    parser._key = key.encode('utf-8')

    raw_dump = redis.execute_command('dump', key)
    if not raw_dump:
        sys.stderr.write('Key %s does not exist\n' % key)
        sys.exit(-1)

    stream = BytesIO(raw_dump)
    data_type = read_unsigned_char(stream)
    parser.read_object(stream, data_type)
 def parse_rdb_to_mysql(cls,
                        rdb_file_path,
                        mysql_server,
                        mysql_table_name,
                        min_key_length,
                        mysql_batch_insert_rows=100,
                        mysql_batch_sleep_seconds=0.01):
     redis_arch = 64
     string_escape = "raw"
     stream_handler = MySQLCallback(
         mysql_server=mysql_server,
         mysql_table_name=mysql_table_name,
         mysql_batch_insert_rows=mysql_batch_insert_rows,
         mysql_batch_sleep_seconds=mysql_batch_sleep_seconds,
         min_key_length=min_key_length)
     stream_handler.init_env()
     parser = RdbParser(
         MemoryCallback(stream_handler,
                        redis_arch,
                        string_escape=string_escape))
     parser.parse(rdb_file_path)
Beispiel #7
0
def main():
    usage = """usage: %(prog)s [options] /path/to/dump.rdb

Example : %(prog)s --command json -k "user.*" /var/redis/6379/dump.rdb"""

    parser = ArgumentParser(prog='rdb', usage=usage)
    parser.add_argument("-c", "--command", dest="command", required=True,
                  help="Command to execute. Valid commands are json, diff, justkeys, justkeyvals, memory and protocol", metavar="CMD")
    parser.add_argument("-f", "--file", dest="output",
                  help="Output file", metavar="FILE")
    parser.add_argument("-n", "--db", dest="dbs", action="append",
                  help="Database Number. Multiple databases can be provided. If not specified, all databases will be included.")
    parser.add_argument("-k", "--key", dest="keys", default=None,
                  help="Keys to export. This can be a regular expression")
    parser.add_argument("-o", "--not-key", dest="not_keys", default=None,
                  help="Keys Not to export. This can be a regular expression")
    parser.add_argument("-t", "--type", dest="types", action="append",
                  help="""Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. 
                    If not specified, all data types will be returned""")
    parser.add_argument("-b", "--bytes", dest="bytes", default=None,
                  help="Limit memory output to keys greater to or equal to this value (in bytes)")
    parser.add_argument("-l", "--largest", dest="largest", default=None,
                  help="Limit memory output to only the top N keys (by size)")
    parser.add_argument("-e", "--escape", dest="escape", choices=ESCAPE_CHOICES,
                  help="Escape strings to encoding: %s (default), %s, %s, or %s." % tuple(ESCAPE_CHOICES))
    expire_group = parser.add_mutually_exclusive_group(required=False)
    expire_group.add_argument("-x", "--no-expire", dest="no_expire", default=False, action='store_true',
                  help="With protocol command, remove expiry from all keys")
    expire_group.add_argument("-a", "--amend-expire", dest="amend_expire", default=0, type=int, metavar='N',
                  help="With protocol command, add N seconds to key expiry time")
    parser.add_argument("dump_file", nargs=1, help="RDB Dump file to process")

    options = parser.parse_args()
    
    filters = {}
    if options.dbs:
        filters['dbs'] = []
        for x in options.dbs:
            try:
                filters['dbs'].append(int(x))
            except ValueError:
                raise Exception('Invalid database number %s' %x)
    
    if options.keys:
        filters['keys'] = options.keys
        
    if options.not_keys:
        filters['not_keys'] = options.not_keys
    
    if options.types:
        filters['types'] = []
        for x in options.types:
            if not x in VALID_TYPES:
                raise Exception('Invalid type provided - %s. Expected one of %s' % (x, (", ".join(VALID_TYPES))))
            else:
                filters['types'].append(x)

    out_file_obj = None
    try:
        if options.output:
            out_file_obj = open(options.output, "wb")
        else:
            # Prefer not to depend on Python stdout implementation for writing binary.
            out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb')

        try:
            callback = {
                'diff': lambda f: DiffCallback(f, string_escape=options.escape),
                'json': lambda f: JSONCallback(f, string_escape=options.escape),
                'justkeys': lambda f: KeysOnlyCallback(f, string_escape=options.escape),
                'justkeyvals': lambda f: KeyValsOnlyCallback(f, string_escape=options.escape),
                'memory': lambda f: MemoryCallback(PrintAllKeys(f, options.bytes, options.largest),
                                                   64, string_escape=options.escape),
                'protocol': lambda f: ProtocolCallback(f, string_escape=options.escape,
                                                       emit_expire=not options.no_expire,
                                                       amend_expire=options.amend_expire
                                                      )
            }[options.command](out_file_obj)
        except:
            raise Exception('Invalid Command %s' % options.command)

        if not PYTHON_LZF_INSTALLED:
            eprint("WARNING: python-lzf package NOT detected. " +
                "Parsing dump file will be very slow unless you install it. " +
                "To install, run the following command:")
            eprint("")
            eprint("pip install python-lzf")
            eprint("")

        parser = RdbParser(callback, filters=filters)
        parser.parse(options.dump_file[0])
    finally:
        if options.output and out_file_obj is not None:
            out_file_obj.close()
Beispiel #8
0
def get_stats(file_name):
    stats = Stats()
    callback = MemoryCallback(stats, 64)
    parser = RdbParser(callback)
    parser.parse(os.path.join(os.path.dirname(__file__), 'dumps', file_name))
    return stats.records
Beispiel #9
0
def main():
    usage = """usage: %prog [options] /path/to/dump.rdb

Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb"""

    parser = OptionParser(usage=usage)
    parser.add_option(
        "-c",
        "--command",
        dest="command",
        help="Command to execute. Valid commands are json or diff",
        metavar="FILE")

    parser.add_option("-f",
                      "--file",
                      dest="output",
                      help="Output file",
                      metavar="FILE")
    parser.add_option(
        "-n",
        "--db",
        dest="dbs",
        action="append",
        help=
        "Database Number. Multiple databases can be provided. If not specified, all databases will be included."
    )
    parser.add_option("-k",
                      "--key",
                      dest="keys",
                      default=None,
                      help="Keys to export. This can be a regular expression")
    parser.add_option(
        "-t",
        "--type",
        dest="types",
        action="append",
        help=
        """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. 
                    If not specified, all data types will be returned""")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.error("Redis RDB file not specified")
    dump_file = args[0]

    filters = {}
    if options.dbs:
        filters['dbs'] = []
        for x in options.dbs:
            try:
                filters['dbs'].append(int(x))
            except ValueError:
                raise Exception('Invalid database number %s' % x)

    if options.keys:
        filters['keys'] = options.keys

    if options.types:
        filters['types'] = []
        for x in options.types:
            if not x in VALID_TYPES:
                raise Exception(
                    'Invalid type provided - %s. Expected one of %s' %
                    (x, (", ".join(VALID_TYPES))))
            else:
                filters['types'].append(x)

    destination = sys.stdout
    if options.output:
        desitnaion = open(options.output, 'wb')

    cmds = {
        'diff': DiffCallback,
        'json': JSONCallback,
        'memory': lambda r: MemoryCallback(r, 64)
    }
    for key, cb in cmds.items():
        if key != options.command:
            continue
        with destination as f:
            parser = RdbParser(cb(f), filters=filters)
            parser.parse(dump_file)
        return True
    raise Exception('Invalid Caommand %s' % options.output)
Beispiel #10
0
def main():
    usage = """usage: %prog [options] /path/to/dump.rdb

Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb"""

    parser = OptionParser(usage=usage)
    parser.add_option(
        "-c",
        "--command",
        dest="command",
        help=
        "Command to execute. Valid commands are json, diff, justkeys, justkeyvals and protocol",
        metavar="FILE")
    parser.add_option("-f",
                      "--file",
                      dest="output",
                      help="Output file",
                      metavar="FILE")
    parser.add_option(
        "-n",
        "--db",
        dest="dbs",
        action="append",
        help=
        "Database Number. Multiple databases can be provided. If not specified, all databases will be included."
    )
    parser.add_option("-k",
                      "--key",
                      dest="keys",
                      default=None,
                      help="Keys to export. This can be a regular expression")
    parser.add_option(
        "-o",
        "--not-key",
        dest="not_keys",
        default=None,
        help="Keys Not to export. This can be a regular expression")
    parser.add_option(
        "-t",
        "--type",
        dest="types",
        action="append",
        help=
        """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. 
                    If not specified, all data types will be returned""")
    parser.add_option(
        "-b",
        "--bytes",
        dest="bytes",
        default=None,
        help=
        "Limit memory output to keys greater to or equal to this value (in bytes)"
    )
    parser.add_option(
        "-l",
        "--largest",
        dest="largest",
        default=None,
        help="Limit memory output to only the top N keys (by size)")
    parser.add_option(
        "-e",
        "--escape",
        dest="escape",
        choices=ESCAPE_CHOICES,
        help="Escape strings to encoding: %s (default), %s, %s, or %s." %
        tuple(ESCAPE_CHOICES))

    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.error("Redis RDB file not specified")
    dump_file = args[0]

    filters = {}
    if options.dbs:
        filters['dbs'] = []
        for x in options.dbs:
            try:
                filters['dbs'].append(int(x))
            except ValueError:
                raise Exception('Invalid database number %s' % x)

    if options.keys:
        filters['keys'] = options.keys

    if options.not_keys:
        filters['not_keys'] = options.not_keys

    if options.types:
        filters['types'] = []
        for x in options.types:
            if not x in VALID_TYPES:
                raise Exception(
                    'Invalid type provided - %s. Expected one of %s' %
                    (x, (", ".join(VALID_TYPES))))
            else:
                filters['types'].append(x)

    out_file_obj = None
    try:
        if options.output:
            out_file_obj = open(options.output, "wb")
        else:
            # Prefer not to depend on Python stdout implementation for writing binary.
            out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb')

        try:
            callback = {
                'diff':
                lambda f: DiffCallback(f, string_escape=options.escape),
                'json':
                lambda f: JSONCallback(f, string_escape=options.escape),
                'justkeys':
                lambda f: KeysOnlyCallback(f, string_escape=options.escape),
                'justkeyvals':
                lambda f: KeyValsOnlyCallback(f, string_escape=options.escape),
                'memory':
                lambda f: MemoryCallback(PrintAllKeys(f, options.bytes, options
                                                      .largest),
                                         64,
                                         string_escape=options.escape),
                'protocol':
                lambda f: ProtocolCallback(f, string_escape=options.escape)
            }[options.command](out_file_obj)
        except:
            raise Exception('Invalid Command %s' % options.command)

        parser = RdbParser(callback, filters=filters)
        parser.parse(dump_file)
    finally:
        if options.output and out_file_obj is not None:
            out_file_obj.close()
Beispiel #11
0
def main():
    usage = """usage: %prog [options] /path/to/dump.rdb

Example : %prog --command json -k "user.*" /var/redis/6379/dump.rdb"""

    parser = OptionParser(usage=usage)
    parser.add_option(
        "-c",
        "--command",
        dest="command",
        help="Command to execute. Valid commands are json or diff",
        metavar="FILE")

    parser.add_option("-f",
                      "--file",
                      dest="output",
                      help="Output file",
                      metavar="FILE")
    parser.add_option(
        "-n",
        "--db",
        dest="dbs",
        action="append",
        help=
        "Database Number. Multiple databases can be provided. If not specified, all databases will be included."
    )
    parser.add_option("-k",
                      "--key",
                      dest="keys",
                      default=None,
                      help="Keys to export. This can be a regular expression")
    parser.add_option(
        "-t",
        "--type",
        dest="types",
        action="append",
        help=
        """Data types to include. Possible values are string, hash, set, sortedset, list. Multiple typees can be provided. 
                    If not specified, all data types will be returned""")

    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.error("Redis RDB file not specified")
    dump_file = args[0]

    filters = {}
    if options.dbs:
        filters['dbs'] = []
        for x in options.dbs:
            try:
                filters['dbs'].append(int(x))
            except ValueError:
                raise Exception('Invalid database number %s' % x)

    if options.keys:
        filters['keys'] = options.keys

    if options.types:
        filters['types'] = []
        for x in options.types:
            if not x in VALID_TYPES:
                raise Exception(
                    'Invalid type provided - %s. Expected one of %s' %
                    (x, (", ".join(VALID_TYPES))))
            else:
                filters['types'].append(x)

    # TODO : Fix this ugly if-else code
    if options.output:
        with open(options.output, "wb") as f:
            if 'diff' == options.command:
                callback = DiffCallback(f)
            elif 'json' == options.command:
                callback = JSONCallback(f)
            elif 'memory' == options.command:
                reporter = PrintAllKeys(f)
                callback = MemoryCallback(reporter, 64)
            else:
                raise Exception('Invalid Command %s' % options.output)
            parser = RdbParser(callback)
            parser.parse(dump_file)
    else:
        if 'diff' == options.command:
            callback = DiffCallback(sys.stdout)
        elif 'json' == options.command:
            callback = JSONCallback(sys.stdout)
        elif 'memory' == options.command:
            reporter = PrintAllKeys(sys.stdout)
            callback = MemoryCallback(reporter, 64)
        else:
            raise Exception('Invalid Command %s' % options.output)

        parser = RdbParser(callback, filters=filters)
        parser.parse(dump_file)