def extractFieldsFromResultDocument(self, fields, document): document = DictUtils.KeyDotNotationDict(document) new_document = DictUtils.KeyDotNotationDict() for field in fields: if field not in document: continue new_document[field] = document[field] return new_document
def extractFieldsFromResultDocumentWithMapping(self, field_mapping, document): document = DictUtils.KeyDotNotationDict(document) new_document = DictUtils.KeyDotNotationDict() for source_field, target_field in field_mapping.iteritems(): if source_field not in document: continue new_document[target_field] = document[source_field] return new_document
def requeueEvents(self): input_modules = {} for module_name, module_info in self.lumbermill.modules.items(): instance = module_info['instances'][0] if instance.module_type == "input": input_modules[instance.__class__.__name__] = instance self.logger.warning("Found unfinished events. Requeing...") for key in self.persistence_backend.iterKeys(): if not key.startswith("%s" % self.key_prefix): continue requeue_counter = 0 event = self.persistence_backend.pop(key) if not event: continue if "source_module" not in event.get("lumbermill", {}): self.logger.warning( "Could not requeue event. Source module info not found in event data." ) continue source_module = event["lumbermill"]["source_module"] if source_module not in input_modules: self.logger.error( "Could not requeue event. Module %s not found." % (source_module)) continue requeue_counter += 1 input_modules[source_module].sendEvent( DictUtils.KeyDotNotationDict(event)) self.logger.warning("Done. Requeued %s events." % (requeue_counter)) self.logger.warning( "Note: If more than one gp instance is running, requeued events count may differ from total events." )
def returnSimpleKeyDotDictOnCopy(self): """ When creating a copy of an event, we do not want to store/remove this in/from the persistence backend. When the original event is requeued it will pass through all modules again, thus creating the same copies etc. A removeFromPersistenceBackendOnGarbageCollect would fail anyways, since the new key is unknown to the backend. Still we can skip removing for speedups. """ new_dict = DictUtils.KeyDotNotationDict() new_dict.__init__ = DictUtils.KeyDotNotationDict.___init___ new_dict.__del__ = DictUtils.KeyDotNotationDict.__del__ new_dict.update( copy.deepcopy(super(DictUtils.KeyDotNotationDict, self))) if "event_id" in new_dict.get("lumbermill", {}): new_dict['lumbermill'][ 'event_id'] = "%032x" % random.getrandbits(128) return new_dict