def load(line):
# getProcessableMessageList already fetch messages with the same
# group_method_id, so what remains to be filtered on are path, method_id
# and signature
path = line.path
method_id = line.method_id
key = path, method_id
uid = line.uid
original_uid = path_and_method_id_dict.get(key)
if original_uid is None:
m = Message.load(line.message, uid=uid, line=line)
try:
result = activity_tool.SQLJoblib_selectDuplicatedLineList(
path=path,
method_id=method_id,
group_method_id=line.group_method_id,
signature=line.signature)
reserve_uid_list = uid_list = [x.uid for x in result]
if reserve_uid_list:
activity_tool.SQLBase_reserveMessageList(
table=self.sql_table,
processing_node=processing_node,
uid=reserve_uid_list)
except:
self._log(WARNING,
'getDuplicateMessageUidList got an exception')
raise
if uid_list:
self._log(TRACE,
'Reserved duplicate messages: %r' % uid_list)
path_and_method_id_dict[key] = uid
return m, uid, uid_list
# We know that original_uid != uid because caller skips lines we returned
# earlier.
return None, original_uid, [uid]
def _validate(self,
activity_tool,
method_id=None,
message_uid=None,
path=None,
tag=None,
serialization_tag=None):
if isinstance(method_id, str):
method_id = [method_id]
if isinstance(path, str):
path = [path]
if isinstance(tag, str):
tag = [tag]
if method_id or message_uid or path or tag or serialization_tag:
result = activity_tool.SQLBase_validateMessageList(
table=self.sql_table,
method_id=method_id,
message_uid=message_uid,
path=path,
tag=tag,
count=False,
serialization_tag=serialization_tag)
message_list = []
for line in result:
m = Message.load(line.message,
line=line,
uid=line.uid,
date=line.date,
processing_node=line.processing_node)
if not hasattr(m, 'order_validation_text'): # BBB
m.order_validation_text = self.getOrderValidationText(m)
message_list.append(m)
return message_list
def load(line):
# getProcessableMessageList already fetch messages with the same
# group_method_id, so what remains to be filtered on are path, method_id
# and signature
path = line.path
method_id = line.method_id
key = path, method_id
uid = line.uid
original_uid = path_and_method_id_dict.get(key)
if original_uid is None:
m = Message.load(line.message, uid=uid, line=line)
try:
# Select duplicates.
result = db.query("SELECT uid FROM message_job"
" WHERE processing_node = 0 AND path = %s AND signature = %s"
" AND method_id = %s AND group_method_id = %s FOR UPDATE" % (
quote(path), quote(line.signature),
quote(method_id), quote(line.group_method_id),
), 0)[1]
uid_list = [x for x, in result]
if uid_list:
self.assignMessageList(db, processing_node, uid_list)
else:
db.query("COMMIT") # XXX: useful ?
except:
self._log(WARNING, 'Failed to reserve duplicates')
db.query("ROLLBACK")
raise
if uid_list:
self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list)
path_and_method_id_dict[key] = uid
return m, uid, uid_list
# We know that original_uid != uid because caller skips lines we returned
# earlier.
return None, original_uid, [uid]
def _validate(self, activity_tool, method_id=None, message_uid=None, path=None, tag=None,
serialization_tag=None):
if isinstance(method_id, str):
method_id = [method_id]
if isinstance(path, str):
path = [path]
if isinstance(tag, str):
tag = [tag]
if method_id or message_uid or path or tag or serialization_tag:
result = activity_tool.SQLBase_validateMessageList(table=self.sql_table,
method_id=method_id,
message_uid=message_uid,
path=path,
tag=tag,
count=False,
serialization_tag=serialization_tag)
message_list = []
for line in result:
m = Message.load(line.message,
line=line,
uid=line.uid,
date=line.date,
processing_node=line.processing_node)
if not hasattr(m, 'order_validation_text'): # BBB
m.order_validation_text = self.getOrderValidationText(m)
message_list.append(m)
return message_list
def distribute(self, activity_tool, node_count):
db = activity_tool.getSQLConnection()
now_date = getNow(db)
where_kw = {
'processing_node': -1,
'to_date': now_date,
'count': READ_MESSAGE_LIMIT,
}
validated_count = 0
while 1:
result = self._getMessageList(db, **where_kw)
if not result:
return
transaction.commit()
validation_text_dict = {'none': 1}
message_dict = {}
for line in result:
message = Message.load(line.message, uid=line.uid, line=line)
if not hasattr(message, 'order_validation_text'): # BBB
message.order_validation_text = self.getOrderValidationText(
message)
self.getExecutableMessageList(activity_tool,
message,
message_dict,
validation_text_dict,
now_date=now_date)
transaction.commit()
if message_dict:
distributable_uid_set = set()
serialization_tag_dict = {}
for message in message_dict.itervalues():
serialization_tag = message.activity_kw.get(
'serialization_tag')
if serialization_tag is None:
distributable_uid_set.add(message.uid)
else:
serialization_tag_dict.setdefault(
serialization_tag, []).append(message)
for message_list in serialization_tag_dict.itervalues():
# Sort list of messages to validate the message with highest score
message_list.sort(key=sort_message_key)
distributable_uid_set.add(message_list[0].uid)
group_method_id = message_list[0].line.group_method_id
if group_method_id == '\0':
continue
for message in message_list[1:]:
if group_method_id == message.line.group_method_id:
distributable_uid_set.add(message.uid)
distributable_count = len(distributable_uid_set)
if distributable_count:
self.assignMessageList(db, 0, distributable_uid_set)
validated_count += distributable_count
if validated_count >= MAX_VALIDATED_LIMIT:
return
where_kw['from_priority'] = line.priority
where_kw['from_date'] = line.date
where_kw['above_uid'] = line.uid
def flush(self,
activity_tool,
object_path,
invoke=0,
method_id=None,
**kw):
"""
object_path is a tuple
"""
path = '/'.join(object_path)
if invoke:
invoked = set()
def invoke(message):
try:
key = self.generateMessageUID(message)
if key in invoked:
return
invoked.add(key)
except AttributeError:
pass
line = getattr(message, 'line', None)
validate_value = VALID if line and line.processing_node != -1 else \
message.validate(self, activity_tool)
if validate_value == VALID:
# Try to invoke the message - what happens if invoke calls flushActivity ??
with ActivityRuntimeEnvironment(message):
activity_tool.invoke(message)
if message.getExecutionState() != MESSAGE_EXECUTED:
raise ActivityFlushError('Could not invoke %s on %s' %
(message.method_id, path))
elif validate_value is INVALID_PATH:
raise ActivityFlushError('The document %s does not exist' %
path)
else:
raise ActivityFlushError('Could not validate %s on %s' %
(message.method_id, path))
for m in activity_tool.getRegisteredMessageList(self):
if object_path == m.object_path and (method_id is None
or method_id == m.method_id):
if invoke:
invoke(m)
activity_tool.unregisterMessage(self, m)
uid_list = []
for line in self._getMessageList(activity_tool,
path=path,
processing=0,
**({
'method_id': method_id
} if method_id else {})):
uid_list.append(line.uid)
if invoke:
invoke(Message.load(line.message, uid=line.uid, line=line))
if uid_list:
activity_tool.SQLBase_delMessage(table=self.sql_table,
uid=uid_list)
def dumpMessageList(self, activity_tool):
# Dump all messages in the table.
message_list = []
dumpMessageList = getattr(activity_tool, 'SQLQueue_dumpMessageList', None)
if dumpMessageList is not None:
result = dumpMessageList()
for line in result:
m = Message.load(line.message, uid=line.uid, line=line)
message_list.append(m)
return message_list
def dumpMessageList(self, activity_tool):
# Dump all messages in the table.
message_list = []
dumpMessageList = getattr(activity_tool, 'SQLQueue_dumpMessageList', None)
if dumpMessageList is not None:
result = dumpMessageList()
for line in result:
m = Message.load(line.message, uid=line.uid, line=line)
message_list.append(m)
return message_list
def flush(self,
activity_tool,
object_path,
invoke=0,
method_id=None,
only_safe=False,
**kw):
"""
object_path is a tuple
"""
path = '/'.join(object_path)
if invoke:
invoked = set()
def invoke(message):
try:
key = self.generateMessageUID(message)
if key in invoked:
return
invoked.add(key)
except AttributeError:
pass
line = getattr(message, 'line', None)
if (line and line.processing_node != -1
or not activity_tool.getDependentMessageList(message)):
# Try to invoke the message - what happens if invoke calls flushActivity ??
with ActivityRuntimeEnvironment(message):
activity_tool.invoke(message)
if message.getExecutionState() != MESSAGE_EXECUTED:
raise ActivityFlushError('Could not invoke %s on %s' %
(message.method_id, path))
else:
raise ActivityFlushError('Could not validate %s on %s' %
(message.method_id, path))
for m in activity_tool.getRegisteredMessageList(self):
if object_path == m.object_path and (method_id is None
or method_id == m.method_id):
if invoke:
invoke(m)
activity_tool.unregisterMessage(self, m)
uid_list = []
db = activity_tool.getSQLConnection()
for line in self._getMessageList(db,
path=path,
**({
'method_id': method_id
} if method_id else {})):
if only_safe and line.processing_node > -2:
continue
uid_list.append(line.uid)
if invoke and line.processing_node <= 0:
invoke(Message.load(line.message, uid=line.uid, line=line))
if uid_list:
self.deleteMessageList(db, uid_list)
def getMessageList(self, activity_tool, *args, **kw):
result = self._getMessageList(activity_tool.getSQLConnection(), *args, **kw)
if type(result) is str: # src__ == 1
return result,
class_name = self.__class__.__name__
return [Message.load(line.message,
activity=class_name,
uid=line.uid,
processing_node=line.processing_node,
retry=line.retry)
for line in result]
def getMessageList(self, *args, **kw):
result = self._getMessageList(*args, **kw)
if type(result) is str: # src__ == 1
return result,
class_name = self.__class__.__name__
return [Message.load(line.message,
activity=class_name,
uid=line.uid,
processing_node=line.processing_node,
retry=line.retry,
processing=line.processing)
for line in result]
def distribute(self, activity_tool, node_count):
offset = 0
assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None)
if assignMessage is not None:
now_date = self.getNow(activity_tool)
validated_count = 0
while 1:
result = self._getMessageList(activity_tool, processing_node=-1,
to_date=now_date, processing=0,
offset=offset, count=READ_MESSAGE_LIMIT)
if not result:
return
transaction.commit()
validation_text_dict = {'none': 1}
message_dict = {}
for line in result:
message = Message.load(line.message, uid=line.uid, line=line)
if not hasattr(message, 'order_validation_text'): # BBB
message.order_validation_text = self.getOrderValidationText(message)
self.getExecutableMessageList(activity_tool, message, message_dict,
validation_text_dict, now_date=now_date)
if message_dict:
distributable_uid_set = set()
serialization_tag_dict = {}
for message in message_dict.itervalues():
serialization_tag = message.activity_kw.get('serialization_tag')
if serialization_tag is None:
distributable_uid_set.add(message.uid)
else:
serialization_tag_dict.setdefault(serialization_tag,
[]).append(message)
for message_list in serialization_tag_dict.itervalues():
# Sort list of messages to validate the message with highest score
message_list.sort(key=sort_message_key)
distributable_uid_set.add(message_list[0].uid)
group_method_id = message_list[0].line.group_method_id
if group_method_id == '\0':
continue
for message in message_list[1:]:
if group_method_id == message.line.group_method_id:
distributable_uid_set.add(message.uid)
distributable_count = len(distributable_uid_set)
if distributable_count:
assignMessage(table=self.sql_table,
processing_node=0, uid=tuple(distributable_uid_set))
validated_count += distributable_count
if validated_count >= MAX_VALIDATED_LIMIT:
return
offset += READ_MESSAGE_LIMIT
def distribute(self, activity_tool, node_count):
offset = 0
assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None)
if assignMessage is not None:
now_date = self.getNow(activity_tool)
validated_count = 0
while 1:
result = self._getMessageList(activity_tool, processing_node=-1,
to_date=now_date, processing=0,
offset=offset, count=READ_MESSAGE_LIMIT)
if not result:
return
transaction.commit()
validation_text_dict = {'none': 1}
message_dict = {}
for line in result:
message = Message.load(line.message, uid=line.uid, line=line)
if not hasattr(message, 'order_validation_text'): # BBB
message.order_validation_text = self.getOrderValidationText(message)
self.getExecutableMessageList(activity_tool, message, message_dict,
validation_text_dict, now_date=now_date)
if message_dict:
distributable_uid_set = set()
serialization_tag_dict = {}
for message in message_dict.itervalues():
serialization_tag = message.activity_kw.get('serialization_tag')
if serialization_tag is None:
distributable_uid_set.add(message.uid)
else:
serialization_tag_dict.setdefault(serialization_tag,
[]).append(message)
for message_list in serialization_tag_dict.itervalues():
# Sort list of messages to validate the message with highest score
message_list.sort(key=sort_message_key)
distributable_uid_set.add(message_list[0].uid)
group_method_id = message_list[0].line.group_method_id
if group_method_id == '\0':
continue
for message in message_list[1:]:
if group_method_id == message.line.group_method_id:
distributable_uid_set.add(message.uid)
distributable_count = len(distributable_uid_set)
if distributable_count:
assignMessage(table=self.sql_table,
processing_node=0, uid=tuple(distributable_uid_set))
validated_count += distributable_count
if validated_count >= MAX_VALIDATED_LIMIT:
return
offset += READ_MESSAGE_LIMIT
def flush(self, activity_tool, object_path, invoke=0, method_id=None, **kw):
"""
object_path is a tuple
"""
path = '/'.join(object_path)
if invoke:
invoked = set()
def invoke(message):
try:
key = self.generateMessageUID(message)
if key in invoked:
return
invoked.add(key)
except AttributeError:
pass
line = getattr(message, 'line', None)
validate_value = VALID if line and line.processing_node != -1 else \
message.validate(self, activity_tool)
if validate_value == VALID:
# Try to invoke the message - what happens if invoke calls flushActivity ??
activity_tool.invoke(message)
if message.getExecutionState() != MESSAGE_EXECUTED:
raise ActivityFlushError('Could not invoke %s on %s'
% (message.method_id, path))
elif validate_value is INVALID_PATH:
raise ActivityFlushError('The document %s does not exist' % path)
else:
raise ActivityFlushError('Could not validate %s on %s'
% (message.method_id, path))
for m in activity_tool.getRegisteredMessageList(self):
if object_path == m.object_path and (
method_id is None or method_id == m.method_id):
if invoke:
invoke(m)
activity_tool.unregisterMessage(self, m)
uid_list = []
for line in self._getMessageList(activity_tool, path=path, processing=0,
**({'method_id': method_id} if method_id else {})):
uid_list.append(line.uid)
if invoke:
invoke(Message.load(line.message, uid=line.uid, line=line))
if uid_list:
activity_tool.SQLBase_delMessage(table=self.sql_table, uid=uid_list)
def load(line): uid = line.uid m = Message.load(line.message, uid=uid, line=line) return m, uid, ()
def distribute(self, activity_tool, node_count):
offset = 0
assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None)
if assignMessage is not None:
now_date = self.getNow(activity_tool)
validated_count = 0
while 1:
result = self._getMessageList(activity_tool, processing_node=-1,
to_date=now_date,
offset=offset, count=READ_MESSAGE_LIMIT)
if not result:
return
transaction.commit()
validation_text_dict = {'none': 1}
message_dict = {}
for line in result:
message = Message.load(line.message, uid=line.uid, line=line)
if not hasattr(message, 'order_validation_text'): # BBB
message.order_validation_text = self.getOrderValidationText(message)
self.getExecutableMessageList(activity_tool, message, message_dict,
validation_text_dict, now_date=now_date)
if message_dict:
message_unique_dict = {}
serialization_tag_dict = {}
distributable_uid_set = set()
deletable_uid_list = []
# remove duplicates
# SQLDict considers object_path, method_id, tag to unify activities,
# but ignores method arguments. They are outside of semantics.
for message in message_dict.itervalues():
message_unique_dict.setdefault(self.generateMessageUID(message),
[]).append(message)
for message_list in message_unique_dict.itervalues():
if len(message_list) > 1:
# Sort list of duplicates to keep the message with highest score
message_list.sort(key=sort_message_key)
deletable_uid_list += [m.uid for m in message_list[1:]]
message = message_list[0]
serialization_tag = message.activity_kw.get('serialization_tag')
if serialization_tag is None:
distributable_uid_set.add(message.uid)
else:
serialization_tag_dict.setdefault(serialization_tag,
[]).append(message)
# Don't let through if there is the same serialization tag in the
# message dict. If there is the same serialization tag, only one can
# be validated and others must wait.
# But messages with group_method_id are exceptions. serialization_tag
# does not stop validating together. Because those messages should
# be processed together at once.
for message_list in serialization_tag_dict.itervalues():
# Sort list of messages to validate the message with highest score
message_list.sort(key=sort_message_key)
distributable_uid_set.add(message_list[0].uid)
group_method_id = message_list[0].line.group_method_id
if group_method_id == '\0':
continue
for message in message_list[1:]:
if group_method_id == message.line.group_method_id:
distributable_uid_set.add(message.uid)
if deletable_uid_list:
activity_tool.SQLBase_delMessage(table=self.sql_table,
uid=deletable_uid_list)
distributable_count = len(distributable_uid_set)
if distributable_count:
assignMessage(table=self.sql_table,
processing_node=0, uid=tuple(distributable_uid_set))
validated_count += distributable_count
if validated_count >= MAX_VALIDATED_LIMIT:
return
offset += READ_MESSAGE_LIMIT
def distribute(self, activity_tool, node_count):
offset = 0
assignMessage = getattr(activity_tool, 'SQLBase_assignMessage', None)
if assignMessage is not None:
now_date = self.getNow(activity_tool)
validated_count = 0
while 1:
result = self._getMessageList(activity_tool,
processing_node=-1,
to_date=now_date,
offset=offset,
count=READ_MESSAGE_LIMIT)
if not result:
return
transaction.commit()
validation_text_dict = {'none': 1}
message_dict = {}
for line in result:
message = Message.load(line.message,
uid=line.uid,
line=line)
if not hasattr(message, 'order_validation_text'): # BBB
message.order_validation_text = self.getOrderValidationText(
message)
self.getExecutableMessageList(activity_tool,
message,
message_dict,
validation_text_dict,
now_date=now_date)
if message_dict:
message_unique_dict = {}
serialization_tag_dict = {}
distributable_uid_set = set()
deletable_uid_list = []
# remove duplicates
# SQLDict considers object_path, method_id, tag to unify activities,
# but ignores method arguments. They are outside of semantics.
for message in message_dict.itervalues():
message_unique_dict.setdefault(
self.generateMessageUID(message),
[]).append(message)
for message_list in message_unique_dict.itervalues():
if len(message_list) > 1:
# Sort list of duplicates to keep the message with highest score
message_list.sort(key=sort_message_key)
deletable_uid_list += [
m.uid for m in message_list[1:]
]
message = message_list[0]
serialization_tag = message.activity_kw.get(
'serialization_tag')
if serialization_tag is None:
distributable_uid_set.add(message.uid)
else:
serialization_tag_dict.setdefault(
serialization_tag, []).append(message)
# Don't let through if there is the same serialization tag in the
# message dict. If there is the same serialization tag, only one can
# be validated and others must wait.
# But messages with group_method_id are exceptions. serialization_tag
# does not stop validating together. Because those messages should
# be processed together at once.
for message_list in serialization_tag_dict.itervalues():
# Sort list of messages to validate the message with highest score
message_list.sort(key=sort_message_key)
distributable_uid_set.add(message_list[0].uid)
group_method_id = message_list[0].line.group_method_id
if group_method_id == '\0':
continue
for message in message_list[1:]:
if group_method_id == message.line.group_method_id:
distributable_uid_set.add(message.uid)
if deletable_uid_list:
activity_tool.SQLBase_delMessage(
table=self.sql_table, uid=deletable_uid_list)
distributable_count = len(distributable_uid_set)
if distributable_count:
assignMessage(table=self.sql_table,
processing_node=0,
uid=tuple(distributable_uid_set))
validated_count += distributable_count
if validated_count >= MAX_VALIDATED_LIMIT:
return
offset += READ_MESSAGE_LIMIT
def load(line):
# getProcessableMessageList already fetch messages with the same
# group_method_id, so what remains to be filtered on are path and
# method_id.
# XXX: What about tag ?
path = line.path
method_id = line.method_id
key = path, method_id
uid = line.uid
original_uid = path_and_method_id_dict.get(key)
if original_uid is None:
m = Message.load(line.message, uid=uid, line=line)
merge_parent = m.activity_kw.get('merge_parent')
try:
if merge_parent:
path_list = []
while merge_parent != path:
path = path.rsplit('/', 1)[0]
assert path
original_uid = path_and_method_id_dict.get(
(path, method_id))
if original_uid is not None:
return None, original_uid, [uid]
path_list.append(path)
uid_list = []
if path_list:
result = activity_tool.SQLDict_selectParentMessage(
path=path_list,
method_id=method_id,
group_method_id=line.group_method_id,
processing_node=processing_node)
if result: # found a parent
# mark child as duplicate
uid_list.append(uid)
# switch to parent
line = result[0]
key = line.path, method_id
uid = line.uid
m = Message.load(line.message,
uid=uid,
line=line)
# return unreserved similar children
result = activity_tool.SQLDict_selectChildMessageList(
path=line.path,
method_id=method_id,
group_method_id=line.group_method_id)
reserve_uid_list = [x.uid for x in result]
uid_list += reserve_uid_list
if not line.processing_node:
# reserve found parent
reserve_uid_list.append(uid)
else:
result = activity_tool.SQLDict_selectDuplicatedLineList(
path=path,
method_id=method_id,
group_method_id=line.group_method_id)
reserve_uid_list = uid_list = [x.uid for x in result]
if reserve_uid_list:
activity_tool.SQLDict_reserveDuplicatedLineList(
processing_node=processing_node,
uid=reserve_uid_list)
else:
activity_tool.SQLDict_commit() # release locks
except:
self._log(WARNING,
'getDuplicateMessageUidList got an exception')
activity_tool.SQLDict_rollback() # release locks
raise
if uid_list:
self._log(TRACE,
'Reserved duplicate messages: %r' % uid_list)
path_and_method_id_dict[key] = uid
return m, uid, uid_list
# We know that original_uid != uid because caller skips lines we returned
# earlier.
return None, original_uid, [uid]
def load(line):
uid = line.uid
m = Message.load(line.message, uid=uid, line=line)
return m, uid, ()
def load(line):
# getProcessableMessageList already fetch messages with the same
# group_method_id, so what remains to be filtered on are path and
# method_id.
# XXX: What about tag ?
path = line.path
method_id = line.method_id
key = path, method_id
uid = line.uid
original_uid = path_and_method_id_dict.get(key)
if original_uid is None:
m = Message.load(line.message, uid=uid, line=line)
merge_parent = m.activity_kw.get('merge_parent')
try:
if merge_parent:
path_list = []
while merge_parent != path:
path = path.rsplit('/', 1)[0]
assert path
original_uid = path_and_method_id_dict.get((path, method_id))
if original_uid is not None:
return None, original_uid, [uid]
path_list.append(path)
uid_list = []
if path_list:
result = activity_tool.SQLDict_selectParentMessage(
path=path_list,
method_id=method_id,
group_method_id=line.group_method_id,
processing_node=processing_node)
if result: # found a parent
# mark child as duplicate
uid_list.append(uid)
# switch to parent
line = result[0]
key = line.path, method_id
uid = line.uid
m = Message.load(line.message, uid=uid, line=line)
# return unreserved similar children
result = activity_tool.SQLDict_selectChildMessageList(
path=line.path,
method_id=method_id,
group_method_id=line.group_method_id)
reserve_uid_list = [x.uid for x in result]
uid_list += reserve_uid_list
if not line.processing_node:
# reserve found parent
reserve_uid_list.append(uid)
else:
result = activity_tool.SQLDict_selectDuplicatedLineList(
path=path,
method_id=method_id,
group_method_id=line.group_method_id)
reserve_uid_list = uid_list = [x.uid for x in result]
if reserve_uid_list:
activity_tool.SQLDict_reserveDuplicatedLineList(
processing_node=processing_node, uid=reserve_uid_list)
else:
activity_tool.SQLDict_commit() # release locks
except:
self._log(WARNING, 'getDuplicateMessageUidList got an exception')
activity_tool.SQLDict_rollback() # release locks
raise
if uid_list:
self._log(TRACE, 'Reserved duplicate messages: %r' % uid_list)
path_and_method_id_dict[key] = uid
return m, uid, uid_list
# We know that original_uid != uid because caller skips lines we returned
# earlier.
return None, original_uid, [uid]
def load(line):
# getProcessableMessageList already fetch messages with the same
# group_method_id, so what remains to be filtered on are path and
# method_id.
# XXX: What about tag ?
path = line.path
method_id = line.method_id
key = path, method_id
uid = line.uid
original_uid = path_and_method_id_dict.get(key)
if original_uid is None:
sql_method_id = " AND method_id = %s AND group_method_id = %s" % (
quote(method_id), quote(line.group_method_id))
m = Message.load(line.message, uid=uid, line=line)
merge_parent = m.activity_kw.get('merge_parent')
try:
if merge_parent:
path_list = []
while merge_parent != path:
path = path.rsplit('/', 1)[0]
assert path
original_uid = path_and_method_id_dict.get(
(path, method_id))
if original_uid is not None:
return None, original_uid, [uid]
path_list.append(path)
uid_list = []
if path_list:
# Select parent messages.
result = Results(
db.query(
"SELECT * FROM message"
" WHERE processing_node IN (0, %s) AND path IN (%s)%s"
" ORDER BY path LIMIT 1 FOR UPDATE" % (
processing_node,
','.join(map(quote, path_list)),
sql_method_id,
), 0))
if result: # found a parent
# mark child as duplicate
uid_list.append(uid)
# switch to parent
line = result[0]
key = line.path, method_id
uid = line.uid
m = Message.load(line.message,
uid=uid,
line=line)
# return unreserved similar children
path = line.path
result = db.query(
"SELECT uid FROM message"
" WHERE processing_node = 0 AND (path = %s OR path LIKE %s)"
"%s FOR UPDATE" % (
quote(path),
quote(path.replace('_', r'\_') + '/%'),
sql_method_id,
), 0)[1]
reserve_uid_list = [x for x, in result]
uid_list += reserve_uid_list
if not line.processing_node:
# reserve found parent
reserve_uid_list.append(uid)
else:
# Select duplicates.
result = db.query(
"SELECT uid FROM message"
" WHERE processing_node = 0 AND path = %s%s FOR UPDATE"
% (
quote(path),
sql_method_id,
), 0)[1]
reserve_uid_list = uid_list = [x for x, in result]
if reserve_uid_list:
self.assignMessageList(db, processing_node,
reserve_uid_list)
else:
db.query("COMMIT") # XXX: useful ?
except:
self._log(WARNING, 'Failed to reserve duplicates')
db.query("ROLLBACK")
raise
if uid_list:
self._log(TRACE,
'Reserved duplicate messages: %r' % uid_list)
path_and_method_id_dict[key] = uid
return m, uid, uid_list
# We know that original_uid != uid because caller skips lines we returned
# earlier.
return None, original_uid, [uid]
