def __key_proc__(self , _keys , _size):
for key in _keys:
if self.__srcr.type(key) == 'string' or self.__tgtr.type(key) == 'string':
self.__string__(key)
elif self.__srcr.type(key) == 'hash' or self.__tgtr.type(key) == 'hash':
self.__hashmap__(key , self.__fieldKey)
else:
getLogger().info('지원하지 않는 type입니다.-- {}'.format(self.__srcr.type(key)))
def setTypeMode(self , _type , _job):
if self.__confDict == None:
raise Exception('Config is Null')
getLogger().debug(type(self.__confDict))
if _type not in self.__confDict['types']:
raise Exception('support type ::'+str(self.__confDict['types']))
if _job not in self.__confDict['jobmodes']:
raise Exception('support job ::'+str(self.__confDict['jobmodes']))
self.__type = _type
self.__mode = _job
self.__bSet = True
self.__connectionRedisServer__()
return self
def __connectionRedisServer__(self):
# Redis Connection
if self.__type == 'rtoc':
srcHost = self.__confDict['srchosts']['host0']['host']
srcPort = self.__confDict['srchosts']['host0']['port']
srcPwd = None
if self.__confDict['srcpwd'] != None and len(self.__confDict['srcpwd']) > 0:
srcPwd = self.__confDict['srcpwd']
self.__srcr = redis.StrictRedis( srcHost , port = srcPort , charset='utf-8' , db = 0 , password = srcPwd)
startup_nodes = []
tgt_hosts_dict = self.__confDict['tgthosts']
getLogger().debug('keys ::{}'.format(tgt_hosts_dict.keys()))
for key in tgt_hosts_dict.keys():
d = {}
d['host'] = tgt_hosts_dict[key]['host']
d['port'] = int(tgt_hosts_dict[key]['port'])
startup_nodes.append(d)
getLogger().debug('startup_nodes:{}'.format(startup_nodes))
self.__tgtr = StrictRedisCluster(startup_nodes = startup_nodes , decode_responses = False)
elif self.__type == 'rtor':
# Source Redis
srcHost = self.__confDict['srchosts']['host0']['host']
srcPort = self.__confDict['srchosts']['host0']['port']
srcPwd = None
if self.__confDict['srcpwd'] != None and len(self.__confDict['srcpwd']) > 0:
srcPwd = self.__confDict['srcpwd']
self.__srcr = redis.StrictRedis( srcHost , port = srcPort , db = 0 , password = srcPwd)
# Target Redis
tgtHost = self.__confDict['tgthosts']['host0']['host']
tgtPort = self.__confDict['tgthosts']['host0']['port']
tgtPwd = None
if self.__confDict['tgtpwd'] != None and len(self.__confDict['tgtpwd']) > 0:
tgtPwd = self.__confDict['tgtpwd']
self.__tgtr = redis.StrictRedis( tgtHost , port = tgtPort , db = 0 , password = tgtPwd)
elif self.__type == 'rtorp':
pass
else:
pass
def getOption():
global parser
usage = """ %prog [option]
ex1) %prog -t [rtoc|rtor|rtorp] -m [copy|compare|delete] -k [key] -f [Field Key]
"""
parser = OptionParser(usage = usage)
parser.add_option('-t' , '--type' , dest = 'type' , help = 'rtoc , rtor , rtorp' , default = None)
parser.add_option('-m' , '--mode' , dest = 'mode' , help = 'copy , compare , delete' , default = None)
parser.add_option('-k' , '--key' , dest = 'key' , help = 'Redis Key' , default = None)
parser.add_option('-f' , '--fkey' , dest = 'fkey' , help = 'Field Key In HashMap' , default = None)
(options , args) = parser.parse_args()
getLogger().debug('args len ::{}'.format(len(args)))
getLogger().debug('options ::{}'.format(options.__str__))
if options.type == None or options.mode == None:
parser.print_help()
sys.exit(0)
def __hashmap__(self , _key , _fkey = None):
keyLen = int(self.__srcr.hlen(_key)) if _fkey == None else 1
keys = self.__srcr.hkeys(_key) if _fkey == None else list(_fkey)
getLogger().info('----hash key:{} , size:{} ----'.format(_key , keyLen))
procs = []
try:
# Mode Select ( copy , compare )
cur_func = None
if self.__mode == 'copy':
cur_func = self.__h_insert__
elif self.__mode == 'compare':
cur_func = self.__h_comp__
elif self.__mode == 'delete':
cur_func = self.__key_delete__
else:
raise Exception('unknown Mode : {}'.format(self.__mode))
if self.__mode == 'delete':
cur_func(_key)
else:
if keyLen >= MRedisManager.divideCount: #MP
bit = int(keyLen / MRedisManager.nProcessCount)
index = 0
for index in range(0 , MRedisManager.nProcessCount):
start = int(index*bit)
end = int(start+bit)
getLogger().debug('--i:{} , s:{} , e:{} , d :{} --'.format(index , start , end , end-keyLen))
if index == (MRedisManager.nProcessCount -1):
end = keyLen
getLogger().debug('--i:{} , s:{} , e:{} , t:{} --'.format(index , start , end , keyLen))
p = Process(target = cur_func , args = (_key , keys[start:end] , self.__srcr , self.__tgtr ))
procs.append(p)
p.start()
#JOINing...
for proc in procs:
proc.join()
else: # No MP
cur_func(_key , keys , self.__srcr , self.__tgtr)
except KeyboardInterrupt:
for proc in procs:
proc.terminate()
def run(self):
if self.__bSet == False:
raise Exception('setTypeMode Method not Called!!')
getLogger().info('-- Start time :{}'.format(getCurrentDefTime()))
if self.__srcr == None or self.__tgtr == None:
raise Exception('Redis Not Connection...')
if self.__key != None:
self.__key_proc__([self.__key] , 1)
else:
srcDbSize = self.__srcr.dbsize()
srcKeys = self.__srcr.keys()
getLogger().info('-- src key size:{}'.format(srcDbSize))
if srcDbSize > MRedisManager.divideCount: # MP
self.__m_key_proc__(srcKeys , srcDbSize)
else:
self.__key_proc__(srcKeys , srcDbSize)
getLogger().info('-- End time :{}'.format(getCurrentDefTime()))
def __copy_string__(self , _key):
v = self.__string_get__(_key , self.__srcr)
if v != None:
getLogger().info('set key:{} , v:{}'.format(_key , v))
self.__tgtr.set(_key ,v)
def __h_insert__(self , _key , _key_list , from_r , to_r):
count = 0
getLogger().info('--pid:{}-- inseart start'.format(os.getpid()))
error_log = []
for fkey in _key_list:
# From GET
v = self.__hash_get__(_key , fkey , from_r)
if v == None:
error_log.append(('%s:%s is error'%(_key , fkey)))
continue
# To Set
to_r.hset(_key , fkey , v)
# Couting...
count+=1
if count%100000 == 0:
getLogger().info('--pid:{}-- insert count'.format(count))
getLogger().info('--pid:{}-- insert end'.format(os.getpid()))
# Error log Write
if len(error_log) > 0:
getLogger().info('--pid:{}-- error log Creating...'.format(os.getpid()))
errWrite = open(os.getcwd()+'/'+str(os.getpid())+'_error.txt' , 'w')
for err in error_log:
fw.write(err.strip()+'\n')
fw.close()
error_log[:]
getLogger().info('--pid:{}-- error log Write Complete...')
else:
getLogger().info('--pid:{}-- Not Error'.format(os.getpid()))
def __h_comp__(self , _key , _key_list , from_r , to_r):
count = 0
getLogger().info('--pid:{}-- compare start'.format(os.getpid()))
error_log = []
same_log = []
for fkey in _key_list:
# From GET
src_v = self.__hash_get__(_key , fkey , from_r)
tgt_v = self.__hash_get__(_key , fkey , to_r)
if src_v == None or tgt_v == None:
error_log.append(('%s:%s is diff'%(_key , fkey)))
continue
if src_v != tgt_v:
error_log.append(('%s:%s is diff'%(_key , fkey)))
continue
same_log.append(('%s:%s is same'%(_key , fkey)))
# Couting...
count+=1
if count % 100000 == 0:
getLogger().info('--pid:{}-- compare count:{}'.format(os.getpid(), count))
getLogger().info('--pid:{}-- compare end'.format(os.getpid()))
# Error log Write
if len(error_log) > 0:
getLogger().info('--pid:{}-- error log Creating...'.format(os.getpid()))
errWrite = open(os.getcwd()+'/'+str(os.getpid())+'_error.txt' , 'w')
for err in error_log:
errWrite.write(err.strip()+'\n')
errWrite.close()
error_log[:]
getLogger().info('--pid:{}-- error log Write Complete...')
else:
getLogger().info('--pid:{}-- Not Error'.format(os.getpid()))
if len(same_log) > 0:
getLogger().info('--pid:{}-- same_log log Creating...'.format(os.getpid()))
sameWrite = open(os.getcwd()+'/'+str(os.getpid())+'_same.txt' , 'w')
for same in same_log:
sameWrite.write(same.strip()+'\n')
sameWrite.close()
same[:]
getLogger().info('--pid:{}-- same log Write Complete...')