def extractFieldsFromResultDocument(self, fields, document):
document = DictUtils.KeyDotNotationDict(document)
new_document = DictUtils.KeyDotNotationDict()
for field in fields:
if field not in document:
continue
new_document[field] = document[field]
return new_document
def extractFieldsFromResultDocumentWithMapping(self, field_mapping,
document):
document = DictUtils.KeyDotNotationDict(document)
new_document = DictUtils.KeyDotNotationDict()
for source_field, target_field in field_mapping.iteritems():
if source_field not in document:
continue
new_document[target_field] = document[source_field]
return new_document
def requeueEvents(self):
input_modules = {}
for module_name, module_info in self.lumbermill.modules.items():
instance = module_info['instances'][0]
if instance.module_type == "input":
input_modules[instance.__class__.__name__] = instance
self.logger.warning("Found unfinished events. Requeing...")
for key in self.persistence_backend.iterKeys():
if not key.startswith("%s" % self.key_prefix):
continue
requeue_counter = 0
event = self.persistence_backend.pop(key)
if not event:
continue
if "source_module" not in event.get("lumbermill", {}):
self.logger.warning(
"Could not requeue event. Source module info not found in event data."
)
continue
source_module = event["lumbermill"]["source_module"]
if source_module not in input_modules:
self.logger.error(
"Could not requeue event. Module %s not found." %
(source_module))
continue
requeue_counter += 1
input_modules[source_module].sendEvent(
DictUtils.KeyDotNotationDict(event))
self.logger.warning("Done. Requeued %s events." % (requeue_counter))
self.logger.warning(
"Note: If more than one gp instance is running, requeued events count may differ from total events."
)
def returnSimpleKeyDotDictOnCopy(self):
"""
When creating a copy of an event, we do not want to store/remove this in/from the persistence backend.
When the original event is requeued it will pass through all modules again, thus creating the
same copies etc.
A removeFromPersistenceBackendOnGarbageCollect would fail anyways, since the new key is unknown to the
backend. Still we can skip removing for speedups.
"""
new_dict = DictUtils.KeyDotNotationDict()
new_dict.__init__ = DictUtils.KeyDotNotationDict.___init___
new_dict.__del__ = DictUtils.KeyDotNotationDict.__del__
new_dict.update(
copy.deepcopy(super(DictUtils.KeyDotNotationDict, self)))
if "event_id" in new_dict.get("lumbermill", {}):
new_dict['lumbermill'][
'event_id'] = "%032x" % random.getrandbits(128)
return new_dict
