def thread_get_shard_data(uid_piece):
begin = time.time()
logging.info('thread_get_block_data uid_piece:{} {} start...'.format(
len(uid_piece), uid_piece[:10]))
uids_lst = get_in_tuple(uid_piece)
query_sql = 'select shard_id,shard_key from pw_user_shard ' \
'where shard_key in {}'.format(uids_lst)
shard_data = {res[0]: res[1] for res in query(query_sql)}
logging.info(
'thread_get_block_data uid_piece:{} {} end. runtime:{}'.format(
len(uid_piece), uid_piece[:10],
time.time() - begin))
return shard_data
def get_block_data(page_size):
logging.info('get_block_data page_size:{} start...'.format(page_size))
begin = time.time()
query_sql = 'select uid,tuid,update_time from pw_block ' \
'order by uid,tuid limit 5000 offset {}'.format(page_size * 5000)
block_uids = set()
results = []
for res in query(query_sql):
results.append(res)
block_uids.add((res[0], res[1]))
logging.info('get_block_data page_size:{} end. runtime:{}'.format(
page_size,
time.time() - begin))
return results, block_uids
def thread_get_block_data(uids_piece):
begin = time.time()
logging.info('thread_get_block_data uid_piece:{} {} start...'.format(
len(uids_piece), uids_piece[:10]))
uids_lst = get_in_tuple(uids_piece)
query_sql = 'select uid,tuid from pw_block ' \
'where uid in {} or tuid in {}'.format(uids_lst, uids_lst)
block_lst = []
for res in query(query_sql):
block_lst.append((res[0], res[1]))
logging.info(
'thread_get_block_data uid_piece:{} {} end. runtime:{}'.format(
len(uids_piece), uids_piece[:10],
time.time() - begin))
return block_lst
def sync_contact_data():
offset = 0
while True:
query_sql = 'select uid,tuid,create_time from pw_contact where state!=1 ' \
'order by uid,tuid limit 5000 offset {}'.format(offset)
frd_lst = []
uids_set = set()
for res in query(query_sql):
frd_pair = res[0], res[1]
frd_lst.append(frd_pair)
uids_set.update(frd_pair)
if not frd_lst:
break
uid_lst = list(uids_set)
mul_get_block_data(uid_lst)
get_shard_data(uid_lst)
offset += 5000