async def process(self, block_num, block_time, trans_id, ops):
global task_type
db = self.db
# print('process %i blcok\'s ops' % block_num)
self.processed_data = {'data': [], 'undo': []}
for op_idx, op in enumerate(ops):
try:
op_type = op[0]
op_detail = op[1]
if op_type == 'vote':
weight = op_detail['weight']
created_at = block_time
updated_at = block_time
voter_id = await self.getId('users', op_detail['voter'])
#print(trans_id, op_idx, 'voter_id:', voter_id)
if voter_id == None:
self.processed_data['undo'].append(
(block_num, trans_id, op_idx, json.dumps(op),
tasks.getTypeId(task_type), block_time))
continue
comment_id = await self.getId(
'comments',
(op_detail['author'], op_detail['permlink']))
#print(trans_id, op_idx, 'post_id:', post_id)
if comment_id == None:
self.processed_data['undo'].append(
(block_num, trans_id, op_idx, json.dumps(op),
tasks.getTypeId(task_type), block_time))
continue
else:
vote_id = await self.getId('comments_votes',
(voter_id, comment_id))
#print(trans_id, op_idx, 'vote_id:', vote_id)
if vote_id != None:
# edit vote
self.processed_data['undo'].append(
(block_num, trans_id, op_idx, json.dumps(op),
tasks.getTypeId(task_type), block_time))
continue
else:
# insert comment vote
if weight >= 0:
updown = True
else:
weight = (-1) * weight
updown = False
self.processed_data['data'].append(
('comment', (comment_id, voter_id, weight,
updown, created_at, updated_at)))
except Exception as e:
self.processed_data['undo'].append(
(block_num, trans_id, op_idx, json.dumps(op),
tasks.getTypeId(task_type), block_time))
utils.PrintException(e)
# print('processed:', self.processed_data)
return self.processed_data
def mainMultiProcess():
global task_type
config = utils.get_config()
while True:
all_tasks = tasks.splitTasks(tasks.get(task_type), config['slice_step'])
if all_tasks != []:
p = ProcessPoolExecutor(config['worker'])
for t in all_tasks:
p.submit(processor, t)
p.shutdown()
time.sleep(3)
async def process(self, block_num, block_time, trans_id, ops):
global task_type
db = self.db
# print('process %i blcok\'s ops' % block_num)
self.processed_data = {'data': [], 'undo': []}
for op_idx, op in enumerate(ops):
try:
op_type = op[0]
op_detail = op[1]
if op_type == 'comment':
json_metadata = op_detail['json_metadata']
try:
json_metadata = json.loads(op_detail['json_metadata'])
except Exception as e:
print('parse json failed:', op_detail['json_metadata'])
continue
comment_id = await self.getId(
'comments',
(op_detail['author'], op_detail['permlink']))
if comment_id == None:
self.processed_data['undo'].append(
(block_num, trans_id, op_idx, json.dumps(op),
tasks.getTypeId(task_type), block_time))
continue
if 'tags' in json_metadata:
if isinstance(json_metadata['tags'], list):
for tag in json_metadata['tags']:
tag_id = await self.getId('tags', tag)
if tag_id != None:
self.processed_data['data'].append(
(comment_id, tag_id))
except Exception as e:
self.processed_data['undo'].append(
(block_num, trans_id, op_idx, json.dumps(op),
tasks.getTypeId(task_type), block_time))
utils.PrintException([block_num, trans_id, op_idx, op])
# print('processed:', self.processed_data)
return self.processed_data
def main():
global task_type
while True:
all_tasks = tasks.get(task_type)
loop_tasks = []
if all_tasks != []:
loop = asyncio.get_event_loop()
for one_task in all_tasks:
user_task = UserProcess(loop, task_type)
loop_tasks.append(
asyncio.ensure_future(user_task.doMultiTasks(one_task)))
loop.run_until_complete(asyncio.wait(loop_tasks))
loop.close()
time.sleep(3)
async def process(self, block_num, block_time, trans_id, ops):
global task_type
db = self.db
# print('process %i blcok\'s ops' % block_num)
self.processed_data = {
'data': [],
'undo': []}
for op_idx, op in enumerate(ops):
try:
op_type = op[0]
op_detail = op[1]
if op_type == 'comment':
created_at = block_time
updated_at = block_time
is_del = False
json_metadata = op_detail['json_metadata']
parent_author_text = op_detail['parent_author']
parent_permlink = op_detail['parent_permlink']
author_text = op_detail['author']
permlink = op_detail['permlink']
title = op_detail['title']
# check if comment edit through body
body = op_detail['body']
dmp = dmp_module.diff_match_patch()
try:
# if patch_fromText successed, this comment is edited.
dmp.patch_fromText(body)
self.processed_data['undo'].append((block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time))
continue
except:
is_exist = await self.checkExist(author_text, permlink)
if is_exist == False:
# this comment is a new comment.
self.processed_data['data'].append((
None, # parent_id
permlink,
title,
body,
json_metadata,
parent_permlink,
created_at,
updated_at,
is_del,
parent_author_text,
author_text))
else:
# this comment is edited and does not use diff_match_patch
self.processed_data['undo'].append((block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time))
continue
elif op_type == 'delete_comment':
self.processed_data['undo'].append((block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time))
print('do_later_del', block_num, trans_id, op_idx)
except Exception as e:
self.processed_data['undo'].append((block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time))
utils.PrintException([block_num, trans_id, op_idx, e])
# print('processed:', self.processed_data)
return self.processed_data
async def process(self, block_num, block_time, trans_id, ops):
global task_type
db = self.db
# print('process %i blcok\'s ops' % block_num, ops)
self.processed_data = {'data': [], 'undo': []}
for op_idx, op in enumerate(ops):
op_type = op[0]
op_detail = op[1]
if op_type == 'custom_json' and 'id' in op_detail and op_detail[
'id'] == 'follow':
if op_detail['json'] == '':
continue
try:
json_data = json.loads(op_detail['json'])
follower = None
following = None
what = None
if isinstance(json_data, dict):
if 'follower' in json_data:
follower = json_data['follower']
if 'following' in json_data:
following = json_data['following']
if 'what' in json_data and isinstance(
json_data['what'],
list) and len(json_data['what']) > 0:
what = json_data['what'][0]
#elif isinstance(json_data, list):
# if len(json_data) >= 2 and json_data[0] == 'follow':
# if 'follower' in json_data[1]:
# follower = json_data[1]['follower']
# if 'following' in json_data[1]:
# following = json_data[1]['following']
# if 'what' in json_data[1] and len(json_data[1]['what']) > 0:
# what = json_data[1]['what'][0]
# else:
# continue
else:
continue
if follower == None and following == None and (
what == None or what == ''):
print('follow_data_error', block_num, trans_id,
follower, following, what, op)
continue
sql = '''
select id, username from users
where username = %s or username = %s'''
cur = await db.cursor()
await cur.execute(sql, (follower, following))
user_data = await cur.fetchall()
await cur.close()
if len(user_data) == 2:
for user in user_data:
if user[1] == follower:
follower_id = user[0]
if user[1] == following:
following_id = user[0]
self.processed_data['data'].append((
follower_id,
following_id,
what,
block_time,
))
else:
self.processed_data['undo'].append(
(block_num, trans_id, op_idx, json.dumps(op),
tasks.getTypeId(task_type), block_time))
except Exception as e:
self.processed_data['undo'].append(
(block_num, trans_id, op_idx, json.dumps(op),
tasks.getTypeId(task_type), block_time))
utils.PrintException([block_num, trans_id, op_idx])
else:
# print('unknown type:', op_type, block_num, trans_id, ops, op_idx)
continue
# print('processed:', self.processed_data)
return self.processed_data