def one_time_migration(file, args_):
redis_host = args_.redis_host
redis_port = args_.redis_port
redis_password = args_.redis_db_password
print("-----One Time Migration Started---------")
start_time = time.time()
print("Dump file download Started")
download_rdb(file, redis_host, redis_port, redis_password)
print("Dump file download Completed")
callback = JSONCallback(0)
print("Parsing Started")
parser_ = RdbParser(callback)
parser_.parse(file)
print("Parsing Completed")
print("Table Creation Started")
tables_ = create_tables_from_stored_rules()
print("Table Creation Completed")
print("Querying...")
data = old_key_value
bulk_insertion(data, tables_)
print("Query Completed")
Session.commit()
print("Target database commit")
end_time = time.time()
print(
"Execution Time For One Time Migration {0} Seconds".format(end_time -
start_time))
return tables_
def main():
parser = optparse.OptionParser()
parser.add_option('-i', '--input', dest='rdb_file', help="input rdb file [REQUIRED]")
parser.add_option('-o', '--output', dest='csv_file', help = "output csv file [REQUIRED]")
parser.add_option('-c', '--column_delimiter', dest='column_delimiter',
help="column delimiter. Default is ctrl-A ('\\001')")
parser.add_option('-l', '--line_delimiter', dest='line_delimiter',
help="row delimiter. Default is newline ('\\n')")
parser.add_option('-p', '--pre_key', dest='pre_key',
help="columns before hash key. Default is empty")
parser.add_option('-P', '--post_key', dest='post_key',
help="columns after hash key. Default is 'views clicks'")
parser.set_defaults(column_delimiter='\001')
parser.set_defaults(line_delimiter='\n')
parser.set_defaults(pre_key='')
parser.set_defaults(post_key='views clicks')
options, remainder = parser.parse_args()
if options.csv_file == None or options.rdb_file == None:
parser.print_help()
sys.exit(1)
outf = open(options.csv_file, 'w')
callback = CsvCallback(outf, options.pre_key.split(), options.post_key.split(),
options.column_delimiter, options.line_delimiter)
parser = RdbParser(callback)
parser.parse(options.rdb_file)
outf.close()
def load_rdb(target, rdb_filename, appkeys):
class MyCallback(RdbCallback) :
''' Simple example to show how callback works.
See RdbCallback for all available callback methods.
See JsonCallback for a concrete example
'''
def set(self, key, value, expiry):
key = key
# print('%s = %s' % (str(key), str(value)))
def hset(self, key, field, value):
key = key
# print('%s.%s = %s' % (str(key), str(field), str(value)))
def sadd(self, key, member):
# print('%s has {%s}' % (str(key), str(member)))
# target.sadd(key, member)
set_op_wrapper(target, "sadd", key, member, appkeys)
def rpush(self, key, value) :
key = key
# print('%s has [%s]' % (str(key), str(value)))
def zadd(self, key, score, member):
key = key
# print('%s has {%s : %s}' % (str(key), str(member), str(score)))
callback = MyCallback()
parser = RdbParser(callback)
parser.parse(rdb_filename)
def get_csv(dump_file_name):
buff = BytesIO()
callback = MemoryCallback(PrintAllKeys(buff, None, None), 64)
parser = RdbParser(callback)
parser.parse(
os.path.join(os.path.dirname(__file__), 'dumps', dump_file_name))
csv = buff.getvalue().decode()
return csv
def load_rdb(rdb_file_name):
output_path = os.path.join(EXPORT_FOLDER, "out.json")
with open(output_path, "w") as out:
callback = JSONCallback(out)
parser = RdbParser(callback)
parser.parse(rdb_file_name)
with open(output_path, "r") as out:
return json.loads(out.read())
def test_skip_expiry(self):
expected = (b'*2\r\n$6\r\nSELECT\r\n$1\r\n0\r\n'
b'*3\r\n$3\r\nSET\r\n$20\r\nexpires_ms_precision\r\n'
b'$27\r\n2022-12-25 10:11:12.573 UTC\r\n')
buf = BytesIO()
parser = RdbParser(ProtocolCallback(buf, emit_expire=False))
parser.parse(self.dumpfile)
self.assertEquals(buf.getvalue(), expected)
def get_csv(dump_file_name):
buff = BytesIO()
callback = MemoryCallback(PrintAllKeys(buff, None, None), 64)
parser = RdbParser(callback)
parser.parse(os.path.join(os.path.dirname(__file__),
'dumps', dump_file_name))
csv = buff.getvalue().decode()
return csv
def test_amend_expiry(self):
expected = (b'*2\r\n$6\r\nSELECT\r\n$1\r\n0\r\n'
b'*3\r\n$3\r\nSET\r\n$20\r\nexpires_ms_precision\r\n'
b'$27\r\n2022-12-25 10:11:12.573 UTC\r\n'
b'*3\r\n$8\r\nEXPIREAT\r\n$20\r\nexpires_ms_precision\r\n'
b'$10\r\n1671965072\r\n')
buf = BytesIO()
parser = RdbParser(ProtocolCallback(buf, amend_expire=2000))
parser.parse(self.dumpfile)
self.assertEquals(buf.getvalue(), expected)
def test_all_dumps(self):
"""Run callback with all test dumps intercepting incidental crashes."""
if self._callback_class is None:
return # Handle unittest discovery attempt to test with this "abstract" class.
for dump_name in glob.glob(os.path.join(os.path.dirname(__file__), TEST_DUMPS_DIR, '*.rdb')):
callback = self._callback_class(out=self._out)
parser = RdbParser(callback)
try:
parser.parse(dump_name)
except Exception as err:
raise self.failureException("%s on %s - %s: %s" % (
self._callback_class.__name__, os.path.basename(dump_name), type(err).__name__, str(err)))
self._out.seek(0)
self._out.truncate()
def rdb_allvalues_concurrency(self, filename):
callback = ParserRDBCallback()
parser = RdbParser(callback)
self.callback_from.show_msg("[%s]parse the rdb start\n" %
(get_time_stamp()))
parser.parse(filename)
head_info_l, hash_keyvalue_l, zset_keyvalue_l, set_keyvalue_l, list_keyvalue_l, str_keyvalue_l = callback.get_values(
)
head_info = "\n".join(str(head) for head in head_info_l)
self.callback_from.show_msg(head_info + "\n")
self.callback_from.show_msg("[%s]end parse \n" % (get_time_stamp()))
self.callback_from.get_end(hash_keyvalue_l, zset_keyvalue_l,
set_keyvalue_l, list_keyvalue_l,
str_keyvalue_l)
def __main__():
total_cores = cpu_count()
#初始化日志服务
cur_path = os.path.dirname(os.path.abspath(__file__))
log_path = os.path.join(cur_path, 'log')
bin_path = os.path.join(cur_path, "bin")
pidfile = os.path.join(bin_path, "server.pid")
cur_pid = os.getpid()
exec_command = "echo %d > %s" % (cur_pid, pidfile)
os.system(exec_command)
if not Logger.init(log_path):
return False
data_file = os.path.join(cur_path, "hash.txt")
#
# if not os.path.exists(data_file):
# Logger.logger.error("file not exists!!")
# return
redis_manage_obj = RedisManage(Logger.logger)
redis_manage_obj.init_redis_pool()
test_callbak = ParseCallback(Logger.logger, redis_manage_obj, data_file)
test_callbak.load_file()
test_callbak.push_test_data_to_redis()
# jobs = list()
# for x in xrange(total_cores):
# p = Process(target = func, args=(Logger.logger,redis_manage_obj))
# p.start()
# jobs.append(p)
#
# for job in jobs:
# job.join()
filters = dict()
filters['types'] = list()
filters['types'].append('hash')
parser = RdbParser(test_callbak, filters=filters)
dump_file = '/data/temp/dump.rdb'
parser.parse(dump_file)
def memory_profiler(dump_file, bytes, largest, output_file=None):
out_file_obj = None
try:
if output_file:
out_file_obj = open(output_file, "wb")
else:
# Prefer not to depend on Python stdout implementation for writing binary.
out_file_obj = os.fdopen(sys.stdout.fileno(), 'wb')
callback = MemoryCallback(MyPrintAllKeys(out_file_obj, bytes, largest),
64,
string_escape=None)
parser = RdbParser(callback, filters={})
parser.parse(dump_file)
finally:
if output_file and out_file_obj is not None:
out_file_obj.close()
class WebAppIntegration:
def __init__(self):
self._parser = Parser()
self._rdbparser = RdbParser(self._parser)
self._rdbparser.parse("dump.rdb")
self._bDict = self._parser.bigDict
self._mydb = False
# conversion table for values
self._conversionTable = {}
self._conversionTable["HEALTHY"] = 0
self._conversionTable["REGISTERING"] = 1
self._conversionTable["IDLE"] = 0
def importInflux(self):
""" Import file to InfluxDB """
cmd = "curl -i -XPOST 'http://*****:*****@influximport.txt"
subprocess.call(cmd, shell=True)
def create_db(self):
""" Create a database called mydb if it doesn't exist """
cmd = "curl -i XPOST http://localhost:8086/query --data-urlencode 'q=CREATE DATABASE mydb'"
if not self._mydb:
subprocess.call(cmd, shell=True)
self._mydb = True
else:
print("MyDB exists.")
def writeToFile(self):
""" Create a file called influximport.txt in compatibility with InfluxDB """
filename = open("influximport.txt", "w")
currentTime = time.time()
for item, value in self._bDict.items():
writeLine = "machine,machine_name=" + item + " "
for kee, val in value.items():
if val in self._conversionTable:
writeLine += kee + "=" + str(self._conversionTable[val])
else:
writeLine += kee + "=" + str(-1)
if kee != list(value.keys())[-1]:
writeLine += ","
writeLine += " " + str(currentTime).split(".")[0]
filename.write(writeLine + "\n")
filename.close()
def test_all_dumps(self):
"""Run callback with all test dumps intercepting incidental crashes."""
if self._callback_class is None:
return # Handle unittest discovery attempt to test with this "abstract" class.
for dump_name in glob.glob(
os.path.join(os.path.dirname(__file__), TEST_DUMPS_DIR,
'*.rdb')):
callback = self._callback_class(out=self._out)
parser = RdbParser(callback)
try:
parser.parse(dump_name)
except Exception as err:
raise self.failureException("%s on %s - %s: %s" %
(self._callback_class.__name__,
os.path.basename(dump_name),
type(err).__name__, str(err)))
self._out.seek(0)
self._out.truncate()
def retrieve_data_at_time(log_folder, time):
"""
Analyze log data for information at a specific time
:param log_folder: Absolute path (to make things easy) of log's data folder
:param time: Time at which to extract information
:return:
"""
rdb_fpath = get_rdb_file(log_folder, time)
parser = RdbParser(JSONCallback(open("ss", "wb")))
return parser.parse(rdb_fpath)
def escape_test_helper(self, escape_name):
if self._callback_class is None:
return # Handle unittest discovery attempt to test with this "abstract" class.
escape = getattr(encodehelpers, escape_name)
callback = self._callback_class(out=self._out, string_escape=escape)
parser = RdbParser(callback)
parser.parse(os.path.join(os.path.dirname(__file__), TEST_DUMPS_DIR, self._fixture['escape_db_file']))
result = self._out.getvalue()
# print('\n%s escape method %s' % (self._callback_class.__name__, escape_name))
# print("\t\tself._fixture['escape_out_%s'] = %s" % (escape, repr(result)))
# try:
# print(result.decode('utf8'))
# except UnicodeDecodeError:
# print(result.decode('latin-1'))
self.assertEqual(result,
self._fixture['escape_out_' + escape],
msg='\n%s escape method %s' % (self._callback_class.__name__, escape_name)
)
def __main__():
#初始化日志服务
cur_path = os.path.dirname(os.path.abspath(__file__))
log_path = os.path.join(cur_path, 'log')
if not Logger.init(log_path):
return False
#callback = JSONCallback(sys.stdout)
test_callbak = ParseCallback(Logger.logger)
filters = dict()
filters['types'] = list()
filters['types'].append('hash')
parser = RdbParser(test_callbak, filters=filters)
dump_file='/data/temp/dump.rdb'
parser.parse(dump_file)
def parse_rdb_to_mysql(cls,
rdb_file_path,
mysql_server,
mysql_table_name,
min_key_length,
mysql_batch_insert_rows=100,
mysql_batch_sleep_seconds=0.01):
redis_arch = 64
string_escape = "raw"
stream_handler = MySQLCallback(
mysql_server=mysql_server,
mysql_table_name=mysql_table_name,
mysql_batch_insert_rows=mysql_batch_insert_rows,
mysql_batch_sleep_seconds=mysql_batch_sleep_seconds,
min_key_length=min_key_length)
stream_handler.init_env()
parser = RdbParser(
MemoryCallback(stream_handler,
redis_arch,
string_escape=string_escape))
parser.parse(rdb_file_path)
def escape_test_helper(self, escape_name):
if self._callback_class is None:
return # Handle unittest discovery attempt to test with this "abstract" class.
escape = getattr(encodehelpers, escape_name)
callback = self._callback_class(out=self._out, string_escape=escape)
parser = RdbParser(callback)
parser.parse(
os.path.join(os.path.dirname(__file__), TEST_DUMPS_DIR,
self._fixture['escape_db_file']))
result = self._out.getvalue()
# print('\n%s escape method %s' % (self._callback_class.__name__, escape_name))
# print("\t\tself._fixture['escape_out_%s'] = %s" % (escape, repr(result)))
# try:
# print(result.decode('utf8'))
# except UnicodeDecodeError:
# print(result.decode('latin-1'))
self.assertEqual(result,
self._fixture['escape_out_' + escape],
msg='\n%s escape method %s' %
(self._callback_class.__name__, escape_name))
def main():
"""
Usage: redis-expired-key-liberator [rdb_path]
"""
print len(sys.argv)
if len(sys.argv) == 1:
rdb_path = '/var/lib/redis/dump-master.rdb'
elif len(sys.argv) == 2:
rdb_path = sys.argv[1]
else:
print 'Usage: redis-expired-key-liberator [rdb_path]'
exit(2)
if not os.path.isfile(rdb_path):
print "Error: could not read file:", rdb_path
exit(1)
callback = KeyCallback()
parser = RdbParser(callback)
parser.parse(rdb_path)
def load_rdb(filename, filters=None):
r = MockRedis()
parser = RdbParser(r, filters)
parser.parse(filename)
return r
def get_stats(file_name):
stats = Stats()
callback = MemoryCallback(stats, 64)
parser = RdbParser(callback)
parser.parse(os.path.join(os.path.dirname(__file__), 'dumps', file_name))
return stats.records
class MyCallback(RdbCallback):
''' Simple example to show how callback works.
See RdbCallback for all available callback methods.
See JsonCallback for a concrete example
'''
def aux_field(self, key, value):
print('aux:[%s:%s]' % (key, value))
def db_size(self, db_size, expires_size):
print('db_size: %s, expires_size %s' % (db_size, expires_size))
def set(self, key, value, expiry, info):
print('%s = %s' % (str(key), str(value)))
def hset(self, key, field, value):
print('%s.%s = %s' % (str(key), str(field), str(value)))
def sadd(self, key, member):
print('%s has {%s}' % (str(key), str(member)))
def rpush(self, key, value):
print('%s has [%s]' % (str(key), str(value)))
def zadd(self, key, score, member):
print('%s has {%s : %s}' % (str(key), str(member), str(score)))
callback = MyCallback(None)
parser = RdbParser(callback)
parser.parse('dump.rdb')
def load_rdb(file_name, filters=None) :
r = MockRedis()
parser = RdbParser(r, filters)
parser.parse(os.path.join(os.path.dirname(__file__), 'dumps', file_name))
return r
import sys
import MySQLdb
from rdbtools import RdbParser, RdbCallback
db = MySQLdb.connect(host="10.73.45.56", user="******", passwd="dddd!!@@##$$", db="shortener")
sur = db.cursor()
sql = "delete from redis"
sur.execute(sql)
class MyCallback(RdbCallback) :
def insert(test,query) :
cur = db.cursor()
cur.execute(query)
db.commit()
def hset(self, key, field, value):
if(key=="URLShortener") :
sql = "INSERT into redis(redis_key, redis_value) values(\'%d\', \'%s\')" % (int(field), value)
self.insert(sql)
callback = MyCallback()
parser = RdbParser(callback)
parser.parse('/home/next/data/dump.rdb')
db.close()
def load_rdb(file_name, filters=None):
r = MockRedis()
parser = RdbParser(r, filters)
parser.parse(os.path.join(os.path.dirname(__file__), 'dumps', file_name))
return r
def batch_migration(args_):
"""
:param args_: command line arguments
:return: None
"""
# First Time CallBack
# do one time migration to create tables
tables = migration.one_time_migration("dump.rdb", args_)
# capture current snapshot of database
migration.download_rdb("./dump.rdb")
callback = migration.JSONCallback(1)
while True:
print("Sleep")
time.sleep(args_.time_interval)
print("-------Batch Migrations Started---------------")
start_time = time.time()
migration.download_rdb("./dump.rdb", args_.redis_host, args_.redis_port, args_.redis_db_password)
parser_ = RdbParser(callback)
parser_.parse("./dump.rdb")
delete_and_update_keys = migration.old_hash_values - migration.new_hash_values
insert_and_update_keys = migration.new_hash_values - migration.old_hash_values
new_key_name = set()
old_key_name = set()
for hash_ in delete_and_update_keys:
key_name = migration.old_hash_table[hash_]
old_key_name.add(key_name)
for hash_ in insert_and_update_keys:
key_name = migration.new_hash_table[hash_]
new_key_name.add(key_name)
deleted_keys = old_key_name - new_key_name
insert_keys = new_key_name - old_key_name
update_keys = new_key_name & old_key_name
print("update keys", len(update_keys))
print("insert keys", len(insert_keys))
print("deleted keys", len(deleted_keys))
keys_delete = {}
for key_name_delete in deleted_keys:
migration.get_dependency_updates(key_name_delete, tables, False)
keys_delete = {key_name_delete: migration.old_key_value[key_name_delete], **keys_delete}
migration.bulk_deletion(keys_delete, tables)
keys = {}
for key_name in insert_keys:
# migration.get_dependency_updates(key_name, tables, True)
keys = {key_name: migration.new_key_value[key_name], **keys}
migration.bulk_insertion(keys, tables)
for key_name in update_keys:
migration.get_dependency_updates(key_name, tables, True)
for table in tables.keys():
regex = tables[table]["regex"]
if re.match(regex, key_name):
if tables[table]["format"] == "multi_row":
update_values = set(migration.new_key_value[key_name]) - set(migration.old_key_value[key_name])
for update_value in update_values:
migration.update({key_name: update_value}, tables[table])
else:
migration.update({key_name: migration.new_key_value[key_name]}, tables[table])
break
migration.Session.commit()
migration.old_hash_values = migration.new_hash_values
migration.old_hash_table = migration.new_hash_table
migration.old_key_value = migration.new_key_value
migration.old_key_name = migration.new_key_name
migration.new_key_name = set()
migration.new_key_value = {}
migration.new_hash_values = set()
migration.new_hash_table = {}
end_time = time.time()
print("Batch Migrations Processing Time {0} Seconds" .format(end_time-start_time))
def main():
global targethost
targethost = sys.argv[1]
for filename in sys.argv[2].split(','):
parser = RdbParser(MyCallback(None))
parser.parse(filename)
def get_stats(file_name):
stats = Stats()
callback = MemoryCallback(stats, 64)
parser = RdbParser(callback)
parser.parse(os.path.join(os.path.dirname(__file__), 'dumps', file_name))
return stats.records
def sadd(self, key, member):
# print "sadd|{}|{}".format(key, member)
self.p.sadd(key, member)
self.execute()
def rpush(self, key, value):
# print "rpush|{}|{}".format(key, value)
self.p.rpush(key, value)
self.execute()
def zadd(self, key, score, member):
# print "zadd|{}|{}|{}".format(key, member, score)
self.p.zadd(key, member, score)
self.execute()
def end_rdb(self):
self.p.execute()
def execute(self):
self.count = (self.count + 1) % 10000
if not self.count:
self.p.execute()
r = redis.Redis.from_url("redis://127.0.0.1:6400")
callback = MyCallback(r)
parser = RdbParser(callback, filters={"dbs": [0]})
print("start {}".format(now()))
parser.parse("/home/xinming.pan/dump.rdb")
print("stop {}".format(now()))
