def load(line): # getProcessableMessageList already fetch messages with the same # group_method_id, so what remains to be filtered on are path, method_id # and signature path = line.path method_id = line.method_id key = path, method_id uid = line.uid original_uid = path_and_method_id_dict.get(key) if original_uid is None: m = Message.load(line.message, uid=uid, line=line) try: result = activity_tool.SQLJoblib_selectDuplicatedLineList( path=path, method_id=method_id, group_method_id=line.group_method_id, signature=line.signature) reserve_uid_list = uid_list = [x.uid for x in result] if reserve_uid_list: activity_tool.SQLBase_reserveMessageList( table=self.sql_table, processing_node=processing_node, uid=reserve_uid_list) except: self._log(WARNING, 'getDuplicateMessageUidList got an exception') raise if uid_list: self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list) path_and_method_id_dict[key] = uid return m, uid, uid_list # We know that original_uid != uid because caller skips lines we returned # earlier. return None, original_uid, [uid]
def _validate(self, activity_tool, method_id=None, message_uid=None, path=None, tag=None, serialization_tag=None): if isinstance(method_id, str): method_id = [method_id] if isinstance(path, str): path = [path] if isinstance(tag, str): tag = [tag] if method_id or message_uid or path or tag or serialization_tag: result = activity_tool.SQLBase_validateMessageList( table=self.sql_table, method_id=method_id, message_uid=message_uid, path=path, tag=tag, count=False, serialization_tag=serialization_tag) message_list = [] for line in result: m = Message.load(line.message, line=line, uid=line.uid, date=line.date, processing_node=line.processing_node) if not hasattr(m, 'order_validation_text'): # BBB m.order_validation_text = self.getOrderValidationText(m) message_list.append(m) return message_list
def load(line): # getProcessableMessageList already fetch messages with the same # group_method_id, so what remains to be filtered on are path, method_id # and signature path = line.path method_id = line.method_id key = path, method_id uid = line.uid original_uid = path_and_method_id_dict.get(key) if original_uid is None: m = Message.load(line.message, uid=uid, line=line) try: # Select duplicates. result = db.query("SELECT uid FROM message_job" " WHERE processing_node = 0 AND path = %s AND signature = %s" " AND method_id = %s AND group_method_id = %s FOR UPDATE" % ( quote(path), quote(line.signature), quote(method_id), quote(line.group_method_id), ), 0)[1] uid_list = [x for x, in result] if uid_list: self.assignMessageList(db, processing_node, uid_list) else: db.query("COMMIT") # XXX: useful ? except: self._log(WARNING, 'Failed to reserve duplicates') db.query("ROLLBACK") raise if uid_list: self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list) path_and_method_id_dict[key] = uid return m, uid, uid_list # We know that original_uid != uid because caller skips lines we returned # earlier. return None, original_uid, [uid]
def _validate(self, activity_tool, method_id=None, message_uid=None, path=None, tag=None, serialization_tag=None): if isinstance(method_id, str): method_id = [method_id] if isinstance(path, str): path = [path] if isinstance(tag, str): tag = [tag] if method_id or message_uid or path or tag or serialization_tag: result = activity_tool.SQLBase_validateMessageList(table=self.sql_table, method_id=method_id, message_uid=message_uid, path=path, tag=tag, count=False, serialization_tag=serialization_tag) message_list = [] for line in result: m = Message.load(line.message, line=line, uid=line.uid, date=line.date, processing_node=line.processing_node) if not hasattr(m, 'order_validation_text'): # BBB m.order_validation_text = self.getOrderValidationText(m) message_list.append(m) return message_list
def distribute(self, activity_tool, node_count): db = activity_tool.getSQLConnection() now_date = getNow(db) where_kw = { 'processing_node': -1, 'to_date': now_date, 'count': READ_MESSAGE_LIMIT, } validated_count = 0 while 1: result = self._getMessageList(db, **where_kw) if not result: return transaction.commit() validation_text_dict = {'none': 1} message_dict = {} for line in result: message = Message.load(line.message, uid=line.uid, line=line) if not hasattr(message, 'order_validation_text'): # BBB message.order_validation_text = self.getOrderValidationText( message) self.getExecutableMessageList(activity_tool, message, message_dict, validation_text_dict, now_date=now_date) transaction.commit() if message_dict: distributable_uid_set = set() serialization_tag_dict = {} for message in message_dict.itervalues(): serialization_tag = message.activity_kw.get( 'serialization_tag') if serialization_tag is None: distributable_uid_set.add(message.uid) else: serialization_tag_dict.setdefault( serialization_tag, []).append(message) for message_list in serialization_tag_dict.itervalues(): # Sort list of messages to validate the message with highest score message_list.sort(key=sort_message_key) distributable_uid_set.add(message_list[0].uid) group_method_id = message_list[0].line.group_method_id if group_method_id == '\0': continue for message in message_list[1:]: if group_method_id == message.line.group_method_id: distributable_uid_set.add(message.uid) distributable_count = len(distributable_uid_set) if distributable_count: self.assignMessageList(db, 0, distributable_uid_set) validated_count += distributable_count if validated_count >= MAX_VALIDATED_LIMIT: return where_kw['from_priority'] = line.priority where_kw['from_date'] = line.date where_kw['above_uid'] = line.uid
def flush(self, activity_tool, object_path, invoke=0, method_id=None, **kw): """ object_path is a tuple """ path = '/'.join(object_path) if invoke: invoked = set() def invoke(message): try: key = self.generateMessageUID(message) if key in invoked: return invoked.add(key) except AttributeError: pass line = getattr(message, 'line', None) validate_value = VALID if line and line.processing_node != -1 else \ message.validate(self, activity_tool) if validate_value == VALID: # Try to invoke the message - what happens if invoke calls flushActivity ?? with ActivityRuntimeEnvironment(message): activity_tool.invoke(message) if message.getExecutionState() != MESSAGE_EXECUTED: raise ActivityFlushError('Could not invoke %s on %s' % (message.method_id, path)) elif validate_value is INVALID_PATH: raise ActivityFlushError('The document %s does not exist' % path) else: raise ActivityFlushError('Could not validate %s on %s' % (message.method_id, path)) for m in activity_tool.getRegisteredMessageList(self): if object_path == m.object_path and (method_id is None or method_id == m.method_id): if invoke: invoke(m) activity_tool.unregisterMessage(self, m) uid_list = [] for line in self._getMessageList(activity_tool, path=path, processing=0, **({ 'method_id': method_id } if method_id else {})): uid_list.append(line.uid) if invoke: invoke(Message.load(line.message, uid=line.uid, line=line)) if uid_list: activity_tool.SQLBase_delMessage(table=self.sql_table, uid=uid_list)
def dumpMessageList(self, activity_tool): # Dump all messages in the table. message_list = [] dumpMessageList = getattr(activity_tool, 'SQLQueue_dumpMessageList', None) if dumpMessageList is not None: result = dumpMessageList() for line in result: m = Message.load(line.message, uid=line.uid, line=line) message_list.append(m) return message_list
def flush(self, activity_tool, object_path, invoke=0, method_id=None, only_safe=False, **kw): """ object_path is a tuple """ path = '/'.join(object_path) if invoke: invoked = set() def invoke(message): try: key = self.generateMessageUID(message) if key in invoked: return invoked.add(key) except AttributeError: pass line = getattr(message, 'line', None) if (line and line.processing_node != -1 or not activity_tool.getDependentMessageList(message)): # Try to invoke the message - what happens if invoke calls flushActivity ?? with ActivityRuntimeEnvironment(message): activity_tool.invoke(message) if message.getExecutionState() != MESSAGE_EXECUTED: raise ActivityFlushError('Could not invoke %s on %s' % (message.method_id, path)) else: raise ActivityFlushError('Could not validate %s on %s' % (message.method_id, path)) for m in activity_tool.getRegisteredMessageList(self): if object_path == m.object_path and (method_id is None or method_id == m.method_id): if invoke: invoke(m) activity_tool.unregisterMessage(self, m) uid_list = [] db = activity_tool.getSQLConnection() for line in self._getMessageList(db, path=path, **({ 'method_id': method_id } if method_id else {})): if only_safe and line.processing_node > -2: continue uid_list.append(line.uid) if invoke and line.processing_node <= 0: invoke(Message.load(line.message, uid=line.uid, line=line)) if uid_list: self.deleteMessageList(db, uid_list)
def getMessageList(self, activity_tool, *args, **kw): result = self._getMessageList(activity_tool.getSQLConnection(), *args, **kw) if type(result) is str: # src__ == 1 return result, class_name = self.__class__.__name__ return [Message.load(line.message, activity=class_name, uid=line.uid, processing_node=line.processing_node, retry=line.retry) for line in result]
def getMessageList(self, *args, **kw): result = self._getMessageList(*args, **kw) if type(result) is str: # src__ == 1 return result, class_name = self.__class__.__name__ return [Message.load(line.message, activity=class_name, uid=line.uid, processing_node=line.processing_node, retry=line.retry, processing=line.processing) for line in result]
def distribute(self, activity_tool, node_count): offset = 0 assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None) if assignMessage is not None: now_date = self.getNow(activity_tool) validated_count = 0 while 1: result = self._getMessageList(activity_tool, processing_node=-1, to_date=now_date, processing=0, offset=offset, count=READ_MESSAGE_LIMIT) if not result: return transaction.commit() validation_text_dict = {'none': 1} message_dict = {} for line in result: message = Message.load(line.message, uid=line.uid, line=line) if not hasattr(message, 'order_validation_text'): # BBB message.order_validation_text = self.getOrderValidationText(message) self.getExecutableMessageList(activity_tool, message, message_dict, validation_text_dict, now_date=now_date) if message_dict: distributable_uid_set = set() serialization_tag_dict = {} for message in message_dict.itervalues(): serialization_tag = message.activity_kw.get('serialization_tag') if serialization_tag is None: distributable_uid_set.add(message.uid) else: serialization_tag_dict.setdefault(serialization_tag, []).append(message) for message_list in serialization_tag_dict.itervalues(): # Sort list of messages to validate the message with highest score message_list.sort(key=sort_message_key) distributable_uid_set.add(message_list[0].uid) group_method_id = message_list[0].line.group_method_id if group_method_id == '\0': continue for message in message_list[1:]: if group_method_id == message.line.group_method_id: distributable_uid_set.add(message.uid) distributable_count = len(distributable_uid_set) if distributable_count: assignMessage(table=self.sql_table, processing_node=0, uid=tuple(distributable_uid_set)) validated_count += distributable_count if validated_count >= MAX_VALIDATED_LIMIT: return offset += READ_MESSAGE_LIMIT
def flush(self, activity_tool, object_path, invoke=0, method_id=None, **kw): """ object_path is a tuple """ path = '/'.join(object_path) if invoke: invoked = set() def invoke(message): try: key = self.generateMessageUID(message) if key in invoked: return invoked.add(key) except AttributeError: pass line = getattr(message, 'line', None) validate_value = VALID if line and line.processing_node != -1 else \ message.validate(self, activity_tool) if validate_value == VALID: # Try to invoke the message - what happens if invoke calls flushActivity ?? activity_tool.invoke(message) if message.getExecutionState() != MESSAGE_EXECUTED: raise ActivityFlushError('Could not invoke %s on %s' % (message.method_id, path)) elif validate_value is INVALID_PATH: raise ActivityFlushError('The document %s does not exist' % path) else: raise ActivityFlushError('Could not validate %s on %s' % (message.method_id, path)) for m in activity_tool.getRegisteredMessageList(self): if object_path == m.object_path and ( method_id is None or method_id == m.method_id): if invoke: invoke(m) activity_tool.unregisterMessage(self, m) uid_list = [] for line in self._getMessageList(activity_tool, path=path, processing=0, **({'method_id': method_id} if method_id else {})): uid_list.append(line.uid) if invoke: invoke(Message.load(line.message, uid=line.uid, line=line)) if uid_list: activity_tool.SQLBase_delMessage(table=self.sql_table, uid=uid_list)
def load(line): uid = line.uid m = Message.load(line.message, uid=uid, line=line) return m, uid, ()
def distribute(self, activity_tool, node_count): offset = 0 assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None) if assignMessage is not None: now_date = self.getNow(activity_tool) validated_count = 0 while 1: result = self._getMessageList(activity_tool, processing_node=-1, to_date=now_date, offset=offset, count=READ_MESSAGE_LIMIT) if not result: return transaction.commit() validation_text_dict = {'none': 1} message_dict = {} for line in result: message = Message.load(line.message, uid=line.uid, line=line) if not hasattr(message, 'order_validation_text'): # BBB message.order_validation_text = self.getOrderValidationText(message) self.getExecutableMessageList(activity_tool, message, message_dict, validation_text_dict, now_date=now_date) if message_dict: message_unique_dict = {} serialization_tag_dict = {} distributable_uid_set = set() deletable_uid_list = [] # remove duplicates # SQLDict considers object_path, method_id, tag to unify activities, # but ignores method arguments. They are outside of semantics. for message in message_dict.itervalues(): message_unique_dict.setdefault(self.generateMessageUID(message), []).append(message) for message_list in message_unique_dict.itervalues(): if len(message_list) > 1: # Sort list of duplicates to keep the message with highest score message_list.sort(key=sort_message_key) deletable_uid_list += [m.uid for m in message_list[1:]] message = message_list[0] serialization_tag = message.activity_kw.get('serialization_tag') if serialization_tag is None: distributable_uid_set.add(message.uid) else: serialization_tag_dict.setdefault(serialization_tag, []).append(message) # Don't let through if there is the same serialization tag in the # message dict. If there is the same serialization tag, only one can # be validated and others must wait. # But messages with group_method_id are exceptions. serialization_tag # does not stop validating together. Because those messages should # be processed together at once. for message_list in serialization_tag_dict.itervalues(): # Sort list of messages to validate the message with highest score message_list.sort(key=sort_message_key) distributable_uid_set.add(message_list[0].uid) group_method_id = message_list[0].line.group_method_id if group_method_id == '\0': continue for message in message_list[1:]: if group_method_id == message.line.group_method_id: distributable_uid_set.add(message.uid) if deletable_uid_list: activity_tool.SQLBase_delMessage(table=self.sql_table, uid=deletable_uid_list) distributable_count = len(distributable_uid_set) if distributable_count: assignMessage(table=self.sql_table, processing_node=0, uid=tuple(distributable_uid_set)) validated_count += distributable_count if validated_count >= MAX_VALIDATED_LIMIT: return offset += READ_MESSAGE_LIMIT
def distribute(self, activity_tool, node_count): offset = 0 assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None) if assignMessage is not None: now_date = self.getNow(activity_tool) validated_count = 0 while 1: result = self._getMessageList(activity_tool, processing_node=-1, to_date=now_date, offset=offset, count=READ_MESSAGE_LIMIT) if not result: return transaction.commit() validation_text_dict = {'none': 1} message_dict = {} for line in result: message = Message.load(line.message, uid=line.uid, line=line) if not hasattr(message, 'order_validation_text'): # BBB message.order_validation_text = self.getOrderValidationText( message) self.getExecutableMessageList(activity_tool, message, message_dict, validation_text_dict, now_date=now_date) if message_dict: message_unique_dict = {} serialization_tag_dict = {} distributable_uid_set = set() deletable_uid_list = [] # remove duplicates # SQLDict considers object_path, method_id, tag to unify activities, # but ignores method arguments. They are outside of semantics. for message in message_dict.itervalues(): message_unique_dict.setdefault( self.generateMessageUID(message), []).append(message) for message_list in message_unique_dict.itervalues(): if len(message_list) > 1: # Sort list of duplicates to keep the message with highest score message_list.sort(key=sort_message_key) deletable_uid_list += [ m.uid for m in message_list[1:] ] message = message_list[0] serialization_tag = message.activity_kw.get( 'serialization_tag') if serialization_tag is None: distributable_uid_set.add(message.uid) else: serialization_tag_dict.setdefault( serialization_tag, []).append(message) # Don't let through if there is the same serialization tag in the # message dict. If there is the same serialization tag, only one can # be validated and others must wait. # But messages with group_method_id are exceptions. serialization_tag # does not stop validating together. Because those messages should # be processed together at once. for message_list in serialization_tag_dict.itervalues(): # Sort list of messages to validate the message with highest score message_list.sort(key=sort_message_key) distributable_uid_set.add(message_list[0].uid) group_method_id = message_list[0].line.group_method_id if group_method_id == '\0': continue for message in message_list[1:]: if group_method_id == message.line.group_method_id: distributable_uid_set.add(message.uid) if deletable_uid_list: activity_tool.SQLBase_delMessage( table=self.sql_table, uid=deletable_uid_list) distributable_count = len(distributable_uid_set) if distributable_count: assignMessage(table=self.sql_table, processing_node=0, uid=tuple(distributable_uid_set)) validated_count += distributable_count if validated_count >= MAX_VALIDATED_LIMIT: return offset += READ_MESSAGE_LIMIT
def load(line): # getProcessableMessageList already fetch messages with the same # group_method_id, so what remains to be filtered on are path and # method_id. # XXX: What about tag ? path = line.path method_id = line.method_id key = path, method_id uid = line.uid original_uid = path_and_method_id_dict.get(key) if original_uid is None: m = Message.load(line.message, uid=uid, line=line) merge_parent = m.activity_kw.get('merge_parent') try: if merge_parent: path_list = [] while merge_parent != path: path = path.rsplit('/', 1)[0] assert path original_uid = path_and_method_id_dict.get( (path, method_id)) if original_uid is not None: return None, original_uid, [uid] path_list.append(path) uid_list = [] if path_list: result = activity_tool.SQLDict_selectParentMessage( path=path_list, method_id=method_id, group_method_id=line.group_method_id, processing_node=processing_node) if result: # found a parent # mark child as duplicate uid_list.append(uid) # switch to parent line = result[0] key = line.path, method_id uid = line.uid m = Message.load(line.message, uid=uid, line=line) # return unreserved similar children result = activity_tool.SQLDict_selectChildMessageList( path=line.path, method_id=method_id, group_method_id=line.group_method_id) reserve_uid_list = [x.uid for x in result] uid_list += reserve_uid_list if not line.processing_node: # reserve found parent reserve_uid_list.append(uid) else: result = activity_tool.SQLDict_selectDuplicatedLineList( path=path, method_id=method_id, group_method_id=line.group_method_id) reserve_uid_list = uid_list = [x.uid for x in result] if reserve_uid_list: activity_tool.SQLDict_reserveDuplicatedLineList( processing_node=processing_node, uid=reserve_uid_list) else: activity_tool.SQLDict_commit() # release locks except: self._log(WARNING, 'getDuplicateMessageUidList got an exception') activity_tool.SQLDict_rollback() # release locks raise if uid_list: self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list) path_and_method_id_dict[key] = uid return m, uid, uid_list # We know that original_uid != uid because caller skips lines we returned # earlier. return None, original_uid, [uid]
def load(line): # getProcessableMessageList already fetch messages with the same # group_method_id, so what remains to be filtered on are path and # method_id. # XXX: What about tag ? path = line.path method_id = line.method_id key = path, method_id uid = line.uid original_uid = path_and_method_id_dict.get(key) if original_uid is None: m = Message.load(line.message, uid=uid, line=line) merge_parent = m.activity_kw.get('merge_parent') try: if merge_parent: path_list = [] while merge_parent != path: path = path.rsplit('/', 1)[0] assert path original_uid = path_and_method_id_dict.get((path, method_id)) if original_uid is not None: return None, original_uid, [uid] path_list.append(path) uid_list = [] if path_list: result = activity_tool.SQLDict_selectParentMessage( path=path_list, method_id=method_id, group_method_id=line.group_method_id, processing_node=processing_node) if result: # found a parent # mark child as duplicate uid_list.append(uid) # switch to parent line = result[0] key = line.path, method_id uid = line.uid m = Message.load(line.message, uid=uid, line=line) # return unreserved similar children result = activity_tool.SQLDict_selectChildMessageList( path=line.path, method_id=method_id, group_method_id=line.group_method_id) reserve_uid_list = [x.uid for x in result] uid_list += reserve_uid_list if not line.processing_node: # reserve found parent reserve_uid_list.append(uid) else: result = activity_tool.SQLDict_selectDuplicatedLineList( path=path, method_id=method_id, group_method_id=line.group_method_id) reserve_uid_list = uid_list = [x.uid for x in result] if reserve_uid_list: activity_tool.SQLDict_reserveDuplicatedLineList( processing_node=processing_node, uid=reserve_uid_list) else: activity_tool.SQLDict_commit() # release locks except: self._log(WARNING, 'getDuplicateMessageUidList got an exception') activity_tool.SQLDict_rollback() # release locks raise if uid_list: self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list) path_and_method_id_dict[key] = uid return m, uid, uid_list # We know that original_uid != uid because caller skips lines we returned # earlier. return None, original_uid, [uid]
def load(line): # getProcessableMessageList already fetch messages with the same # group_method_id, so what remains to be filtered on are path and # method_id. # XXX: What about tag ? path = line.path method_id = line.method_id key = path, method_id uid = line.uid original_uid = path_and_method_id_dict.get(key) if original_uid is None: sql_method_id = " AND method_id = %s AND group_method_id = %s" % ( quote(method_id), quote(line.group_method_id)) m = Message.load(line.message, uid=uid, line=line) merge_parent = m.activity_kw.get('merge_parent') try: if merge_parent: path_list = [] while merge_parent != path: path = path.rsplit('/', 1)[0] assert path original_uid = path_and_method_id_dict.get( (path, method_id)) if original_uid is not None: return None, original_uid, [uid] path_list.append(path) uid_list = [] if path_list: # Select parent messages. result = Results( db.query( "SELECT * FROM message" " WHERE processing_node IN (0, %s) AND path IN (%s)%s" " ORDER BY path LIMIT 1 FOR UPDATE" % ( processing_node, ','.join(map(quote, path_list)), sql_method_id, ), 0)) if result: # found a parent # mark child as duplicate uid_list.append(uid) # switch to parent line = result[0] key = line.path, method_id uid = line.uid m = Message.load(line.message, uid=uid, line=line) # return unreserved similar children path = line.path result = db.query( "SELECT uid FROM message" " WHERE processing_node = 0 AND (path = %s OR path LIKE %s)" "%s FOR UPDATE" % ( quote(path), quote(path.replace('_', r'\_') + '/%'), sql_method_id, ), 0)[1] reserve_uid_list = [x for x, in result] uid_list += reserve_uid_list if not line.processing_node: # reserve found parent reserve_uid_list.append(uid) else: # Select duplicates. result = db.query( "SELECT uid FROM message" " WHERE processing_node = 0 AND path = %s%s FOR UPDATE" % ( quote(path), sql_method_id, ), 0)[1] reserve_uid_list = uid_list = [x for x, in result] if reserve_uid_list: self.assignMessageList(db, processing_node, reserve_uid_list) else: db.query("COMMIT") # XXX: useful ? except: self._log(WARNING, 'Failed to reserve duplicates') db.query("ROLLBACK") raise if uid_list: self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list) path_and_method_id_dict[key] = uid return m, uid, uid_list # We know that original_uid != uid because caller skips lines we returned # earlier. return None, original_uid, [uid]