async def process(self, block_num, block_time, trans_id, ops): global task_type db = self.db # print('process %i blcok\'s ops' % block_num) self.processed_data = {'data': [], 'undo': []} for op_idx, op in enumerate(ops): try: op_type = op[0] op_detail = op[1] if op_type == 'vote': weight = op_detail['weight'] created_at = block_time updated_at = block_time voter_id = await self.getId('users', op_detail['voter']) #print(trans_id, op_idx, 'voter_id:', voter_id) if voter_id == None: self.processed_data['undo'].append( (block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) continue comment_id = await self.getId( 'comments', (op_detail['author'], op_detail['permlink'])) #print(trans_id, op_idx, 'post_id:', post_id) if comment_id == None: self.processed_data['undo'].append( (block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) continue else: vote_id = await self.getId('comments_votes', (voter_id, comment_id)) #print(trans_id, op_idx, 'vote_id:', vote_id) if vote_id != None: # edit vote self.processed_data['undo'].append( (block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) continue else: # insert comment vote if weight >= 0: updown = True else: weight = (-1) * weight updown = False self.processed_data['data'].append( ('comment', (comment_id, voter_id, weight, updown, created_at, updated_at))) except Exception as e: self.processed_data['undo'].append( (block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) utils.PrintException(e) # print('processed:', self.processed_data) return self.processed_data
def mainMultiProcess(): global task_type config = utils.get_config() while True: all_tasks = tasks.splitTasks(tasks.get(task_type), config['slice_step']) if all_tasks != []: p = ProcessPoolExecutor(config['worker']) for t in all_tasks: p.submit(processor, t) p.shutdown() time.sleep(3)
async def process(self, block_num, block_time, trans_id, ops): global task_type db = self.db # print('process %i blcok\'s ops' % block_num) self.processed_data = {'data': [], 'undo': []} for op_idx, op in enumerate(ops): try: op_type = op[0] op_detail = op[1] if op_type == 'comment': json_metadata = op_detail['json_metadata'] try: json_metadata = json.loads(op_detail['json_metadata']) except Exception as e: print('parse json failed:', op_detail['json_metadata']) continue comment_id = await self.getId( 'comments', (op_detail['author'], op_detail['permlink'])) if comment_id == None: self.processed_data['undo'].append( (block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) continue if 'tags' in json_metadata: if isinstance(json_metadata['tags'], list): for tag in json_metadata['tags']: tag_id = await self.getId('tags', tag) if tag_id != None: self.processed_data['data'].append( (comment_id, tag_id)) except Exception as e: self.processed_data['undo'].append( (block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) utils.PrintException([block_num, trans_id, op_idx, op]) # print('processed:', self.processed_data) return self.processed_data
def main(): global task_type while True: all_tasks = tasks.get(task_type) loop_tasks = [] if all_tasks != []: loop = asyncio.get_event_loop() for one_task in all_tasks: user_task = UserProcess(loop, task_type) loop_tasks.append( asyncio.ensure_future(user_task.doMultiTasks(one_task))) loop.run_until_complete(asyncio.wait(loop_tasks)) loop.close() time.sleep(3)
async def process(self, block_num, block_time, trans_id, ops): global task_type db = self.db # print('process %i blcok\'s ops' % block_num) self.processed_data = { 'data': [], 'undo': []} for op_idx, op in enumerate(ops): try: op_type = op[0] op_detail = op[1] if op_type == 'comment': created_at = block_time updated_at = block_time is_del = False json_metadata = op_detail['json_metadata'] parent_author_text = op_detail['parent_author'] parent_permlink = op_detail['parent_permlink'] author_text = op_detail['author'] permlink = op_detail['permlink'] title = op_detail['title'] # check if comment edit through body body = op_detail['body'] dmp = dmp_module.diff_match_patch() try: # if patch_fromText successed, this comment is edited. dmp.patch_fromText(body) self.processed_data['undo'].append((block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) continue except: is_exist = await self.checkExist(author_text, permlink) if is_exist == False: # this comment is a new comment. self.processed_data['data'].append(( None, # parent_id permlink, title, body, json_metadata, parent_permlink, created_at, updated_at, is_del, parent_author_text, author_text)) else: # this comment is edited and does not use diff_match_patch self.processed_data['undo'].append((block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) continue elif op_type == 'delete_comment': self.processed_data['undo'].append((block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) print('do_later_del', block_num, trans_id, op_idx) except Exception as e: self.processed_data['undo'].append((block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) utils.PrintException([block_num, trans_id, op_idx, e]) # print('processed:', self.processed_data) return self.processed_data
async def process(self, block_num, block_time, trans_id, ops): global task_type db = self.db # print('process %i blcok\'s ops' % block_num, ops) self.processed_data = {'data': [], 'undo': []} for op_idx, op in enumerate(ops): op_type = op[0] op_detail = op[1] if op_type == 'custom_json' and 'id' in op_detail and op_detail[ 'id'] == 'follow': if op_detail['json'] == '': continue try: json_data = json.loads(op_detail['json']) follower = None following = None what = None if isinstance(json_data, dict): if 'follower' in json_data: follower = json_data['follower'] if 'following' in json_data: following = json_data['following'] if 'what' in json_data and isinstance( json_data['what'], list) and len(json_data['what']) > 0: what = json_data['what'][0] #elif isinstance(json_data, list): # if len(json_data) >= 2 and json_data[0] == 'follow': # if 'follower' in json_data[1]: # follower = json_data[1]['follower'] # if 'following' in json_data[1]: # following = json_data[1]['following'] # if 'what' in json_data[1] and len(json_data[1]['what']) > 0: # what = json_data[1]['what'][0] # else: # continue else: continue if follower == None and following == None and ( what == None or what == ''): print('follow_data_error', block_num, trans_id, follower, following, what, op) continue sql = ''' select id, username from users where username = %s or username = %s''' cur = await db.cursor() await cur.execute(sql, (follower, following)) user_data = await cur.fetchall() await cur.close() if len(user_data) == 2: for user in user_data: if user[1] == follower: follower_id = user[0] if user[1] == following: following_id = user[0] self.processed_data['data'].append(( follower_id, following_id, what, block_time, )) else: self.processed_data['undo'].append( (block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) except Exception as e: self.processed_data['undo'].append( (block_num, trans_id, op_idx, json.dumps(op), tasks.getTypeId(task_type), block_time)) utils.PrintException([block_num, trans_id, op_idx]) else: # print('unknown type:', op_type, block_num, trans_id, ops, op_idx) continue # print('processed:', self.processed_data) return self.processed_data