Beispiel #1
0
  def _validate(self, activity_tool, method_id=None, message_uid=None, path=None, tag=None,
                serialization_tag=None):
    if isinstance(method_id, str):
      method_id = [method_id]
    if isinstance(path, str):
      path = [path]
    if isinstance(tag, str):
      tag = [tag]

    if method_id or message_uid or path or tag or serialization_tag:
      result = activity_tool.SQLBase_validateMessageList(table=self.sql_table,
          method_id=method_id,
          message_uid=message_uid,
          path=path,
          tag=tag,
          count=False,
          serialization_tag=serialization_tag)
      message_list = []
      for line in result:
        m = Message.load(line.message,
                         line=line,
                         uid=line.uid,
                         date=line.date,
                         processing_node=line.processing_node)
        if not hasattr(m, 'order_validation_text'): # BBB
          m.order_validation_text = self.getOrderValidationText(m)
        message_list.append(m)
      return message_list
Beispiel #2
0
 def dumpMessageList(self, activity_tool):
   # Dump all messages in the table.
   message_list = []
   dumpMessageList = getattr(activity_tool, 'SQLQueue_dumpMessageList', None)
   if dumpMessageList is not None:
     result = dumpMessageList()
     for line in result:
       m = Message.load(line.message, uid=line.uid, line=line)
       message_list.append(m)
   return message_list
Beispiel #3
0
 def getMessageList(self, *args, **kw):
   result = self._getMessageList(*args, **kw)
   if type(result) is str: # src__ == 1
     return result,
   class_name = self.__class__.__name__
   return [Message.load(line.message,
                            activity=class_name,
                            uid=line.uid,
                            processing_node=line.processing_node,
                            retry=line.retry,
                            processing=line.processing)
     for line in result]
Beispiel #4
0
  def distribute(self, activity_tool, node_count):
    offset = 0
    assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None)
    if assignMessage is not None:
      now_date = self.getNow(activity_tool)
      validated_count = 0
      while 1:
        result = self._getMessageList(activity_tool, processing_node=-1,
                                      to_date=now_date, processing=0,
                                      offset=offset, count=READ_MESSAGE_LIMIT)
        if not result:
          return
        transaction.commit()

        validation_text_dict = {'none': 1}
        message_dict = {}
        for line in result:
          message = Message.load(line.message, uid=line.uid, line=line)
          if not hasattr(message, 'order_validation_text'): # BBB
            message.order_validation_text = self.getOrderValidationText(message)
          self.getExecutableMessageList(activity_tool, message, message_dict,
                                        validation_text_dict, now_date=now_date)
        if message_dict:
          distributable_uid_set = set()
          serialization_tag_dict = {}
          for message in message_dict.itervalues():
            serialization_tag = message.activity_kw.get('serialization_tag')
            if serialization_tag is None:
              distributable_uid_set.add(message.uid)
            else:
              serialization_tag_dict.setdefault(serialization_tag,
                                                []).append(message)
          for message_list in serialization_tag_dict.itervalues():
            # Sort list of messages to validate the message with highest score
            message_list.sort(key=sort_message_key)
            distributable_uid_set.add(message_list[0].uid)
            group_method_id = message_list[0].line.group_method_id
            if group_method_id == '\0':
              continue
            for message in message_list[1:]:
              if group_method_id == message.line.group_method_id:
                distributable_uid_set.add(message.uid)
          distributable_count = len(distributable_uid_set)
          if distributable_count:
            assignMessage(table=self.sql_table,
              processing_node=0, uid=tuple(distributable_uid_set))
            validated_count += distributable_count
            if validated_count >= MAX_VALIDATED_LIMIT:
              return
        offset += READ_MESSAGE_LIMIT
Beispiel #5
0
 def flush(self, activity_tool, object_path, invoke=0, method_id=None, **kw):
   """
     object_path is a tuple
   """
   path = '/'.join(object_path)
   if invoke:
     invoked = set()
     def invoke(message):
       try:
         key = self.generateMessageUID(message)
         if key in invoked:
           return
         invoked.add(key)
       except AttributeError:
         pass
       line = getattr(message, 'line', None)
       validate_value = VALID if line and line.processing_node != -1 else \
                        message.validate(self, activity_tool)
       if validate_value == VALID:
         # Try to invoke the message - what happens if invoke calls flushActivity ??
         activity_tool.invoke(message)
         if message.getExecutionState() != MESSAGE_EXECUTED:
           raise ActivityFlushError('Could not invoke %s on %s'
                                    % (message.method_id, path))
       elif validate_value is INVALID_PATH:
         raise ActivityFlushError('The document %s does not exist' % path)
       else:
         raise ActivityFlushError('Could not validate %s on %s'
                                  % (message.method_id, path))
   for m in activity_tool.getRegisteredMessageList(self):
     if object_path == m.object_path and (
        method_id is None or method_id == m.method_id):
       if invoke:
         invoke(m)
       activity_tool.unregisterMessage(self, m)
   uid_list = []
   for line in self._getMessageList(activity_tool, path=path, processing=0,
       **({'method_id': method_id} if method_id else {})):
     uid_list.append(line.uid)
     if invoke:
       invoke(Message.load(line.message, uid=line.uid, line=line))
   if uid_list:
     activity_tool.SQLBase_delMessage(table=self.sql_table, uid=uid_list)
Beispiel #6
0
 def load(line):
   uid = line.uid
   m = Message.load(line.message, uid=uid, line=line)
   return m, uid, ()
Beispiel #7
0
 def load(line):
   uid = line.uid
   m = Message.load(line.message, uid=uid, line=line)
   return m, uid, ()
Beispiel #8
0
      if m.is_registered:
        active_process_uid = m.active_process_uid
        date = m.activity_kw.get('at_date')
        row = ','.join((
          '@uid+%s' % i,
          quote('/'.join(m.object_path)),
          'NULL' if active_process_uid is None else str(active_process_uid),
          "UTC_TIMESTAMP(6)" if date is None else quote(render_datetime(date)),
          quote(m.method_id),
          '-1' if hasDependency(m) else '0',
          str(m.activity_kw.get('priority', 1)),
          str(m.activity_kw.get('node', 0)),
          quote(m.getGroupId()),
          quote(m.activity_kw.get('tag', '')),
          quote(m.activity_kw.get('serialization_tag', '')),
          quote(Message.dump(m))))
        i += 1
        n = sep_len + len(row)
        max_payload -= n
        if max_payload < 0:
          if values_list:
            insert(reset_uid)
            reset_uid = False
            max_payload = self._insert_max_payload - n
          else:
            raise ValueError("max_allowed_packet too small to insert message")
        values_list.append(row)
    if values_list:
      insert(reset_uid)

  def _getMessageList(self, db, count=1000, src__=0, **kw):
Beispiel #9
0
 def load(line):
   # getProcessableMessageList already fetch messages with the same
   # group_method_id, so what remains to be filtered on are path and
   # method_id.
   # XXX: What about tag ?
   path = line.path
   method_id = line.method_id
   key = path, method_id
   uid = line.uid
   original_uid = path_and_method_id_dict.get(key)
   if original_uid is None:
     sql_method_id = " AND method_id = %s AND group_method_id = %s" % (
       quote(method_id), quote(line.group_method_id))
     m = Message.load(line.message, uid=uid, line=line)
     merge_parent = m.activity_kw.get('merge_parent')
     try:
       if merge_parent:
         path_list = []
         while merge_parent != path:
           path = path.rsplit('/', 1)[0]
           assert path
           original_uid = path_and_method_id_dict.get((path, method_id))
           if original_uid is not None:
             return None, original_uid, [uid]
           path_list.append(path)
         uid_list = []
         if path_list:
           # Select parent messages.
           result = Results(db.query("SELECT * FROM message"
             " WHERE processing_node IN (0, %s) AND path IN (%s)%s"
             " ORDER BY path LIMIT 1 FOR UPDATE" % (
               processing_node,
               ','.join(map(quote, path_list)),
               sql_method_id,
             ), 0))
           if result: # found a parent
             # mark child as duplicate
             uid_list.append(uid)
             # switch to parent
             line = result[0]
             key = line.path, method_id
             uid = line.uid
             m = Message.load(line.message, uid=uid, line=line)
         # return unreserved similar children
         path = line.path
         result = db.query("SELECT uid FROM message"
           " WHERE processing_node = 0 AND (path = %s OR path LIKE %s)"
           "%s FOR UPDATE" % (
             quote(path), quote(path.replace('_', r'\_') + '/%'),
             sql_method_id,
           ), 0)[1]
         reserve_uid_list = [x for x, in result]
         uid_list += reserve_uid_list
         if not line.processing_node:
           # reserve found parent
           reserve_uid_list.append(uid)
       else:
         # Select duplicates.
         result = db.query("SELECT uid FROM message"
           " WHERE processing_node = 0 AND path = %s%s FOR UPDATE" % (
             quote(path), sql_method_id,
           ), 0)[1]
         reserve_uid_list = uid_list = [x for x, in result]
       if reserve_uid_list:
         self.assignMessageList(db, processing_node, reserve_uid_list)
       else:
         db.query("COMMIT") # XXX: useful ?
     except:
       self._log(WARNING, 'Failed to reserve duplicates')
       db.query("ROLLBACK")
       raise
     if uid_list:
       self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list)
     path_and_method_id_dict[key] = uid
     return m, uid, uid_list
   # We know that original_uid != uid because caller skips lines we returned
   # earlier.
   return None, original_uid, [uid]
Beispiel #10
0
 def load(line):
     # getProcessableMessageList already fetch messages with the same
     # group_method_id, so what remains to be filtered on are path and
     # method_id.
     # XXX: What about tag ?
     path = line.path
     method_id = line.method_id
     key = path, method_id
     uid = line.uid
     original_uid = path_and_method_id_dict.get(key)
     if original_uid is None:
         m = Message.load(line.message, uid=uid, line=line)
         merge_parent = m.activity_kw.get('merge_parent')
         try:
             if merge_parent:
                 path_list = []
                 while merge_parent != path:
                     path = path.rsplit('/', 1)[0]
                     assert path
                     original_uid = path_and_method_id_dict.get(
                         (path, method_id))
                     if original_uid is not None:
                         return None, original_uid, [uid]
                     path_list.append(path)
                 uid_list = []
                 if path_list:
                     result = activity_tool.SQLDict_selectParentMessage(
                         path=path_list,
                         method_id=method_id,
                         group_method_id=line.group_method_id,
                         processing_node=processing_node)
                     if result:  # found a parent
                         # mark child as duplicate
                         uid_list.append(uid)
                         # switch to parent
                         line = result[0]
                         key = line.path, method_id
                         uid = line.uid
                         m = Message.load(line.message,
                                          uid=uid,
                                          line=line)
                 # return unreserved similar children
                 result = activity_tool.SQLDict_selectChildMessageList(
                     path=line.path,
                     method_id=method_id,
                     group_method_id=line.group_method_id)
                 reserve_uid_list = [x.uid for x in result]
                 uid_list += reserve_uid_list
                 if not line.processing_node:
                     # reserve found parent
                     reserve_uid_list.append(uid)
             else:
                 result = activity_tool.SQLDict_selectDuplicatedLineList(
                     path=path,
                     method_id=method_id,
                     group_method_id=line.group_method_id)
                 reserve_uid_list = uid_list = [x.uid for x in result]
             if reserve_uid_list:
                 activity_tool.SQLDict_reserveDuplicatedLineList(
                     processing_node=processing_node,
                     uid=reserve_uid_list)
             else:
                 activity_tool.SQLDict_commit()  # release locks
         except:
             self._log(WARNING,
                       'getDuplicateMessageUidList got an exception')
             activity_tool.SQLDict_rollback()  # release locks
             raise
         if uid_list:
             self._log(TRACE,
                       'Reserved duplicate messages: %r' % uid_list)
         path_and_method_id_dict[key] = uid
         return m, uid, uid_list
     # We know that original_uid != uid because caller skips lines we returned
     # earlier.
     return None, original_uid, [uid]
Beispiel #11
0
    def distribute(self, activity_tool, node_count):
        offset = 0
        assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None)
        if assignMessage is not None:
            now_date = self.getNow(activity_tool)
            validated_count = 0
            while 1:
                result = self._getMessageList(activity_tool,
                                              processing_node=-1,
                                              to_date=now_date,
                                              offset=offset,
                                              count=READ_MESSAGE_LIMIT)
                if not result:
                    return
                transaction.commit()

                validation_text_dict = {'none': 1}
                message_dict = {}
                for line in result:
                    message = Message.load(line.message,
                                           uid=line.uid,
                                           line=line)
                    if not hasattr(message, 'order_validation_text'):  # BBB
                        message.order_validation_text = self.getOrderValidationText(
                            message)
                    self.getExecutableMessageList(activity_tool,
                                                  message,
                                                  message_dict,
                                                  validation_text_dict,
                                                  now_date=now_date)

                if message_dict:
                    message_unique_dict = {}
                    serialization_tag_dict = {}
                    distributable_uid_set = set()
                    deletable_uid_list = []

                    # remove duplicates
                    # SQLDict considers object_path, method_id, tag to unify activities,
                    # but ignores method arguments. They are outside of semantics.
                    for message in message_dict.itervalues():
                        message_unique_dict.setdefault(
                            self.generateMessageUID(message),
                            []).append(message)
                    for message_list in message_unique_dict.itervalues():
                        if len(message_list) > 1:
                            # Sort list of duplicates to keep the message with highest score
                            message_list.sort(key=sort_message_key)
                            deletable_uid_list += [
                                m.uid for m in message_list[1:]
                            ]
                        message = message_list[0]
                        serialization_tag = message.activity_kw.get(
                            'serialization_tag')
                        if serialization_tag is None:
                            distributable_uid_set.add(message.uid)
                        else:
                            serialization_tag_dict.setdefault(
                                serialization_tag, []).append(message)
                    # Don't let through if there is the same serialization tag in the
                    # message dict. If there is the same serialization tag, only one can
                    # be validated and others must wait.
                    # But messages with group_method_id are exceptions. serialization_tag
                    # does not stop validating together. Because those messages should
                    # be processed together at once.
                    for message_list in serialization_tag_dict.itervalues():
                        # Sort list of messages to validate the message with highest score
                        message_list.sort(key=sort_message_key)
                        distributable_uid_set.add(message_list[0].uid)
                        group_method_id = message_list[0].line.group_method_id
                        if group_method_id == '\0':
                            continue
                        for message in message_list[1:]:
                            if group_method_id == message.line.group_method_id:
                                distributable_uid_set.add(message.uid)
                    if deletable_uid_list:
                        activity_tool.SQLBase_delMessage(
                            table=self.sql_table, uid=deletable_uid_list)
                    distributable_count = len(distributable_uid_set)
                    if distributable_count:
                        assignMessage(table=self.sql_table,
                                      processing_node=0,
                                      uid=tuple(distributable_uid_set))
                        validated_count += distributable_count
                        if validated_count >= MAX_VALIDATED_LIMIT:
                            return
                offset += READ_MESSAGE_LIMIT
Beispiel #12
0
    def distribute(self, activity_tool, node_count):
        offset = 0
        assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None)
        if assignMessage is not None:
            now_date = self.getNow(activity_tool)
            validated_count = 0
            while 1:
                result = self._getMessageList(activity_tool,
                                              processing_node=-1,
                                              to_date=now_date,
                                              processing=0,
                                              offset=offset,
                                              count=READ_MESSAGE_LIMIT)
                if not result:
                    return
                transaction.commit()

                validation_text_dict = {'none': 1}
                message_dict = {}
                for line in result:
                    message = Message.load(line.message,
                                           uid=line.uid,
                                           line=line)
                    if not hasattr(message, 'order_validation_text'):  # BBB
                        message.order_validation_text = self.getOrderValidationText(
                            message)
                    self.getExecutableMessageList(activity_tool,
                                                  message,
                                                  message_dict,
                                                  validation_text_dict,
                                                  now_date=now_date)
                if message_dict:
                    distributable_uid_set = set()
                    serialization_tag_dict = {}
                    for message in message_dict.itervalues():
                        serialization_tag = message.activity_kw.get(
                            'serialization_tag')
                        if serialization_tag is None:
                            distributable_uid_set.add(message.uid)
                        else:
                            serialization_tag_dict.setdefault(
                                serialization_tag, []).append(message)
                    for message_list in serialization_tag_dict.itervalues():
                        # Sort list of messages to validate the message with highest score
                        message_list.sort(key=sort_message_key)
                        distributable_uid_set.add(message_list[0].uid)
                        group_method_id = message_list[0].line.group_method_id
                        if group_method_id == '\0':
                            continue
                        for message in message_list[1:]:
                            if group_method_id == message.line.group_method_id:
                                distributable_uid_set.add(message.uid)
                    distributable_count = len(distributable_uid_set)
                    if distributable_count:
                        assignMessage(table=self.sql_table,
                                      processing_node=0,
                                      uid=tuple(distributable_uid_set))
                        validated_count += distributable_count
                        if validated_count >= MAX_VALIDATED_LIMIT:
                            return
                offset += READ_MESSAGE_LIMIT
Beispiel #13
0
 def load(line):
   # getProcessableMessageList already fetch messages with the same
   # group_method_id, so what remains to be filtered on are path and
   # method_id.
   # XXX: What about tag ?
   path = line.path
   method_id = line.method_id
   key = path, method_id
   uid = line.uid
   original_uid = path_and_method_id_dict.get(key)
   if original_uid is None:
     m = Message.load(line.message, uid=uid, line=line)
     merge_parent = m.activity_kw.get('merge_parent')
     try:
       if merge_parent:
         path_list = []
         while merge_parent != path:
           path = path.rsplit('/', 1)[0]
           assert path
           original_uid = path_and_method_id_dict.get((path, method_id))
           if original_uid is not None:
             return None, original_uid, [uid]
           path_list.append(path)
         uid_list = []
         if path_list:
           result = activity_tool.SQLDict_selectParentMessage(
             path=path_list,
             method_id=method_id,
             group_method_id=line.group_method_id,
             processing_node=processing_node)
           if result: # found a parent
             # mark child as duplicate
             uid_list.append(uid)
             # switch to parent
             line = result[0]
             key = line.path, method_id
             uid = line.uid
             m = Message.load(line.message, uid=uid, line=line)
         # return unreserved similar children
         result = activity_tool.SQLDict_selectChildMessageList(
           path=line.path,
           method_id=method_id,
           group_method_id=line.group_method_id)
         reserve_uid_list = [x.uid for x in result]
         uid_list += reserve_uid_list
         if not line.processing_node:
           # reserve found parent
           reserve_uid_list.append(uid)
       else:
         result = activity_tool.SQLDict_selectDuplicatedLineList(
           path=path,
           method_id=method_id,
           group_method_id=line.group_method_id)
         reserve_uid_list = uid_list = [x.uid for x in result]
       if reserve_uid_list:
         activity_tool.SQLDict_reserveDuplicatedLineList(
           processing_node=processing_node, uid=reserve_uid_list)
       else:
         activity_tool.SQLDict_commit() # release locks
     except:
       self._log(WARNING, 'getDuplicateMessageUidList got an exception')
       activity_tool.SQLDict_rollback() # release locks
       raise
     if uid_list:
       self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list)
     path_and_method_id_dict[key] = uid
     return m, uid, uid_list
   # We know that original_uid != uid because caller skips lines we returned
   # earlier.
   return None, original_uid, [uid]
Beispiel #14
0
  def distribute(self, activity_tool, node_count):
    offset = 0
    assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None)
    if assignMessage is not None:
      now_date = self.getNow(activity_tool)
      validated_count = 0
      while 1:
        result = self._getMessageList(activity_tool, processing_node=-1,
                                      to_date=now_date,
                                      offset=offset, count=READ_MESSAGE_LIMIT)
        if not result:
          return
        transaction.commit()

        validation_text_dict = {'none': 1}
        message_dict = {}
        for line in result:
          message = Message.load(line.message, uid=line.uid, line=line)
          if not hasattr(message, 'order_validation_text'): # BBB
            message.order_validation_text = self.getOrderValidationText(message)
          self.getExecutableMessageList(activity_tool, message, message_dict,
                                        validation_text_dict, now_date=now_date)

        if message_dict:
          message_unique_dict = {}
          serialization_tag_dict = {}
          distributable_uid_set = set()
          deletable_uid_list = []

          # remove duplicates
          # SQLDict considers object_path, method_id, tag to unify activities,
          # but ignores method arguments. They are outside of semantics.
          for message in message_dict.itervalues():
            message_unique_dict.setdefault(self.generateMessageUID(message),
                                           []).append(message)
          for message_list in message_unique_dict.itervalues():
            if len(message_list) > 1:
              # Sort list of duplicates to keep the message with highest score
              message_list.sort(key=sort_message_key)
              deletable_uid_list += [m.uid for m in message_list[1:]]
            message = message_list[0]
            serialization_tag = message.activity_kw.get('serialization_tag')
            if serialization_tag is None:
              distributable_uid_set.add(message.uid)
            else:
              serialization_tag_dict.setdefault(serialization_tag,
                                                []).append(message)
          # Don't let through if there is the same serialization tag in the
          # message dict. If there is the same serialization tag, only one can
          # be validated and others must wait.
          # But messages with group_method_id are exceptions. serialization_tag
          # does not stop validating together. Because those messages should
          # be processed together at once.
          for message_list in serialization_tag_dict.itervalues():
            # Sort list of messages to validate the message with highest score
            message_list.sort(key=sort_message_key)
            distributable_uid_set.add(message_list[0].uid)
            group_method_id = message_list[0].line.group_method_id
            if group_method_id == '\0':
              continue
            for message in message_list[1:]:
              if group_method_id == message.line.group_method_id:
                distributable_uid_set.add(message.uid)
          if deletable_uid_list:
            activity_tool.SQLBase_delMessage(table=self.sql_table,
                                             uid=deletable_uid_list)
          distributable_count = len(distributable_uid_set)
          if distributable_count:
            assignMessage(table=self.sql_table,
              processing_node=0, uid=tuple(distributable_uid_set))
            validated_count += distributable_count
            if validated_count >= MAX_VALIDATED_LIMIT:
              return
        offset += READ_MESSAGE_LIMIT