Example #1
0
    def initialize(self, existingSegment=False, customProcessing=None, setModelMaturity=False):
        """Initialize the consumer, the producer, and start the
        maturity count."""

        self.updator = self.engine.producerUpdateScheme.updator(COUNT)   # use the producer's UpdateScheme
        if not existingSegment:
            self.lock = False
            self.pmmlModelMaturity.attrib["locked"] = False
        else:
            if setModelMaturity or ("updateExisting" in self.producerParameters and self.producerParameters["updateExisting"] is True):
                self.updator.initialize({COUNT: self.pmmlModelMaturity.attrib["numUpdates"]})
            self.lock = self.pmmlModelMaturity.attrib["locked"]

        self.consumerAlgorithm.initialize()
        self.producerAlgorithm.initialize(**self.producerParameters)

        self.constants = self.pmmlModel.child(pmml.Extension, exception=False)
        if self.constants is None:
            self.constants = NameSpaceReadOnly()
        else:
            self.constants = self.constants.child(pmml.X_ODG_CustomProcessingConstants, exception=False)
            if self.constants is None:
                self.constants = NameSpaceReadOnly()
            else:
                self.constants = self.constants.nameSpace

        self.userFriendly = getattr(self, "userFriendly", new.instance(Segment))
        self.userFriendly.name = self.name()
        self.userFriendly.pmmlPredicate = self.pmmlPredicate
        self.userFriendly.expression = self.expressionTree
        self.userFriendly.evaluate = self.predicateMatches
        self.userFriendly.pmmlModel = self.pmmlModel
        self.userFriendly.consumer = self.consumerAlgorithm
        self.userFriendly.producer = self.producerAlgorithm
        self.userFriendly.const = self.constants
        self.userFriendly.state = self.state

        if customProcessing is not None:
            db = customProcessing.persistentStorage.db
            if self.userFriendly.name not in db:
                db[self.userFriendly.name] = NameSpace()
            self.userFriendly.db = db[self.userFriendly.name]
Example #2
0
    def __init__(self, config_file=None, rethrowExceptions=False, dataStream=None):
        # Get the configuration settings (as an XML instance)
        if isinstance(config_file, config.AugustusConfiguration):
            configRoot = config_file
        else:
            configRoot = xmlbase.loadfile(config_file, config.Config, lineNumbers=True)

        if "randomSeed" in configRoot.attrib:
            augustusRandomSeed = configRoot.attrib["randomSeed"]
            random.seed(augustusRandomSeed)
            numpy.random.seed(augustusRandomSeed + 1)
        else:
            augustusRandomSeed = "unspecified"

        setupLogging(configRoot.matches(lambda x: x.tag in ("Logging", "Metadata")))
        logger = logging.getLogger()
        metadata = logging.getLogger("metadata")

        for l in logger, metadata:
            if "initialization" in l.differentLevel:
                l.setLevel(l.differentLevel["initialization"])
            else:
                l.setLevel(l.naturalLevel)

        logger.info("Loading PMML model.")
        modelInput = configRoot.child(config.ModelInput, exception=False)
        metadata.startTiming("Time to load PMML model")
        pmmlModel, pmmlFileName = getModel(modelInput)
        metadata.stopTiming("Time to load PMML model")
        metadata.data["PMML model file"] = pmmlFileName

        logger.info("Setting up data input.")
        child = configRoot.child(config.DataInput, exception=False)
        if dataStream is None:
            fromHTTP = child.child(config.FromHTTP, exception=False)
            if fromHTTP is None:
                dataStreamer = getDataStreamer(child)                
            else:
                dataStreamer = AugustusHTTPDataStream(fromHTTP)
        else:
            dataStreamer = dataStream

        child = configRoot.child(config.ConsumerBlending, exception=False)
        consumerUpdateScheme = getUpdateScheme(child)

        child = configRoot.child(config.ModelSetup, exception=False)
        # Default Model setup parameters
        modelWriter = getModelWriter(None)
        engineSettings = {"maturityThreshold": modelInput.attrib.get("maturityThreshold", 0),
                          "augustusRandomSeed": augustusRandomSeed,
                          "hasProducer": True,
                          }
        filenameOnException = None
        producerUpdateScheme = getUpdateScheme(None)
        segmentationScheme = SegmentationScheme(None, pmmlModel)
        aggregateUpdateFlag = updateFlag = False

        producerAlgorithm = config.producerAlgorithmDefaults
        for pa in producerAlgorithm.values():
            if pa.validate() is not None:
                raise Exception, "Programmer error in producerAlgorithmDefaults"
        if child is not None:
            for pa in child.matches(config.ProducerAlgorithm):
                producerAlgorithm[pa.attrib["model"]] = pa

        # Model setup
        if child:
            logger.info("Setting up model updating/producing.")
            modelWriter = getModelWriter(child)
            segmentationScheme = SegmentationScheme(child.child(config.SegmentationSchema, exception=False), pmmlModel)
            if modelWriter is not None:
                engineSettings["lockAllSegments"] = child.attrib.get("mode", None) == "lockExisting"
                if child.attrib.get("mode", None) == "updateExisting":
                    for pa in producerAlgorithm.values():
                        pa.parameters["updateExisting"] = True
                if child.attrib.get("mode", None) == "replaceExisting":
                    for pa in producerAlgorithm.values():
                        pa.parameters["updateExisting"] = False

                updateFlag = child.attrib.get("updateEvery", "event") in ("event", "both")
                aggregateUpdateFlag = child.attrib.get("updateEvery", "event") in ("aggregate", "both")
                filenameOnException = "".join([modelWriter.baseName, _modelExceptionIdentifier, ".pmml"])
                child = child.child(config.ProducerBlending, exception=False)
                producerUpdateScheme = getUpdateScheme(child)
                if child and child.exists(config.MaturityThreshold):
                    maturityConfig = child.child(config.MaturityThreshold)
                    engineSettings["maturityThreshold"] = int(maturityConfig.attrib.get("threshold", 1))
                    engineSettings["lockingThreshold"] = \
                        None if "lockingThreshold" not in \
                        maturityConfig.attrib \
                        else int(maturityConfig["lockingThreshold"])
                if engineSettings["lockAllSegments"] and segmentationScheme is not None and not segmentationScheme._generic and not segmentationScheme._whiteList:
                    logger.warning("The model is locked and no new segments are specified...new model files will be unchanged.")
            else:
                logger.warning("There is no outputFile attribute in the ModelSetup; no new model file will be created.")
        else:
            engineSettings["hasProducer"] = False

        # Set up output
        child = configRoot.child(config.Output, exception=False)
        outputWriter = getOutputWriter(child, pmmlFileName)
        child = configRoot.child(config.EventSettings, exception=False)
        if child is not None:
            logger.info("Setting up output.")
            # not in a dictionary to reduce the number of lookups while looping
            scoreFlag = child.attrib["score"]
            outputFlag = child.attrib["output"]
        else:
            scoreFlag = outputFlag = False
        child = configRoot.child(config.AggregationSettings, exception=False)
        if child is not None:
            aggregateScoreFlag = child.attrib["score"]
            aggregateOutputFlag = child.attrib["output"]
            aggregationSettings = child.attrib
        else:
            aggregateScoreFlag = False
            aggregateOutputFlag = False
            aggregationSettings = None

        metadata.data["Update model"] = "true" if updateFlag or aggregateUpdateFlag else "false"

        # build engine once without a data stream
        engine = Engine(pmmlModel, None, producerUpdateScheme, consumerUpdateScheme, segmentationScheme, producerAlgorithm, **engineSettings)
        engine.initialize()
        if outputWriter: outputWriter.open()

        for l in logger, metadata:
            if "verification" in l.differentLevel:
                l.eventLogLevel = l.differentLevel["verification"]
                l.setLevel(l.differentLevel["verification"])
            else:
                l.eventLogLevel = l.naturalLevel
                l.setLevel(l.naturalLevel)

        # score fake data from <ModelVerifications>
        modelVerificationConfig = configRoot.child(config.ModelVerification, exception=False)
        if modelVerificationConfig is not None:
            augustus.engine.verification.verify(modelVerificationConfig, engine, logger, outputWriter)

        # verification can increment aggregate variables, but
        # aggregates should all start at zero at the start of real
        # processing, whether verification happened or not
        engine.flushAggregates()

        if isinstance(dataStreamer, AugustusHTTPDataStream):
            if outputWriter is None:
                dataStreamer.respond = False
            if dataStreamer.respond:
                dataStreamer.setupOutput(outputWriter)

        for l in logger, metadata:
            if "eventloop" in l.differentLevel:
                l.eventLogLevel = l.differentLevel["eventloop"]
                l.setLevel(l.differentLevel["eventloop"])
            else:
                l.eventLogLevel = l.naturalLevel
                l.setLevel(l.naturalLevel)

        # possibly set up custom processing
        customProcessing = configRoot.child(config.CustomProcessing, exception=False)
        if customProcessing is not None:
            constants = engine.pmmlModel.child(pmml.Extension, exception=False)
            if constants is None:
                constants = NameSpaceReadOnly()
            else:
                constants = constants.child(pmml.X_ODG_CustomProcessingConstants, exception=False)
                if constants is None:
                    constants = NameSpaceReadOnly()
                else:
                    constants = constants.nameSpace

            atoms = {"INVALID": INVALID, "MISSING": MISSING, "IMMATURE": IMMATURE, "MATURE": MATURE, "LOCKED": LOCKED, "UNINITIALIZED": UNINITIALIZED}
            for thing in pmml.OutputField.__dict__.values() + pmml.X_ODG_OutputField.__dict__.values():
                if isinstance(thing, Atom):
                    atoms[repr(thing)] = thing

            customProcessing.initialize(pmmlModel, engine.pmmlModel, constants, [s.userFriendly for s in engine.segmentRecords], atoms, logger, metadata, consumerUpdateScheme, producerUpdateScheme)
            engine.customProcessing = customProcessing
            engine.reinitialize()

        else:
            # only shut off circular garbage collection if there is no CustomProcessing or AugustusInterface
            gc.disable()

        self.dataStreamer = dataStreamer
        self.logger = logger
        self.engine = engine
        self.metadata = metadata
        self.aggregationSettings = aggregationSettings
        self.rethrowExceptions = rethrowExceptions
        self.scoreFlag = scoreFlag
        self.updateFlag = updateFlag
        self.outputWriter = outputWriter
        self.outputFlag = outputFlag
        self.modelWriter = modelWriter
        self.filenameOnException = filenameOnException
        self.pmmlModel = pmmlModel
        self.aggregateScoreFlag = aggregateScoreFlag
        self.aggregateUpdateFlag = aggregateUpdateFlag
        self.aggregateOutputFlag = aggregateOutputFlag
        self.customProcessing = customProcessing
Example #3
0
class SegmentRecord:
    """Represent a segment from a PMML file (outside of pmml.py).

    Maintains the ProducerAlgorithm, the ConsumerAlgorithm, and a copy
    of the OutputFields (MiningModel's <Output> and the segment's
    <Output> are concatenated).

    Has pointers to just about everything: the PMML snippet, the
    predicate, the engine...

    The maturity counter (maintained by self.updator) represents the
    number of producer calls that were *successful*.
    """

    maturityThreshold = 10
    lockingThreshold = 10
    segmentNameRegistry = SegmentNameRegistry()

    def __init__(self, pmmlModel, pmmlPredicate, parentPmmlOutput, engine, name=None):
        """Called by Engine when a PMML file is loaded or when a new
        segment is observed."""

        self.pmmlModel = pmmlModel

        self.pmmlPredicate = pmmlPredicate
        if pmmlPredicate is not None:
            streamlined = True
            if pmmlPredicate.exists(lambda x: isinstance(x, pmml.CompoundPredicate) and x.attrib["booleanOperator"] == "surrogate", maxdepth=None):
                streamlined = False
            self.predicateMatches = pmmlPredicate.createTest(streamlined)
            self.expressionTree = pmmlPredicate.expressionTree()
        else:
            self.predicateMatches = None
            self.expressionTree = None

        # merge this <Output> section with the parent's
        thisOutput = pmmlModel.child(pmml.Output, exception=False)
        if thisOutput is None:
            self.pmmlOutput = parentPmmlOutput
        elif parentPmmlOutput is None:
            self.pmmlOutput = thisOutput
        else:
            self.pmmlOutput = parentPmmlOutput.copy()
            for outputField in thisOutput.matches(pmml.OutputField):
                self.pmmlOutput.children.append(outputField)
            self.pmmlOutput.validate()

        self.engine = engine
        self.segmentNameRegistry.register(name, id(self))

        # make an X-ODG-ModelMaturity object to keep track of how many updates this segment has seen
        pmmlExtension = self.pmmlModel.child(pmml.Extension, exception=False)
        if pmmlExtension is not None:
            self.pmmlModelMaturity = pmmlExtension.child(pmml.X_ODG_ModelMaturity, exception=False)
            if self.pmmlModelMaturity is None:
                self.pmmlModelMaturity = pmml.X_ODG_ModelMaturity(numUpdates=0, locked=self.engine.lockAllSegments)
                pmmlExtension.children.append(self.pmmlModelMaturity)
            elif self.engine.lockAllSegments:
                # Always lock if the user asked for it in the configuration file
                self.pmmlModelMaturity.attrib["locked"] = True
        else:
            pmmlExtension = pmml.Extension()
            self.pmmlModelMaturity = pmml.X_ODG_ModelMaturity(numUpdates=0, locked=self.engine.lockAllSegments)
            pmmlExtension.children.append(self.pmmlModelMaturity)
            self.pmmlModel.children.insert(0, pmmlExtension)

    def initialize(self, existingSegment=False, customProcessing=None, setModelMaturity=False):
        """Initialize the consumer, the producer, and start the
        maturity count."""

        self.updator = self.engine.producerUpdateScheme.updator(COUNT)   # use the producer's UpdateScheme
        if not existingSegment:
            self.lock = False
            self.pmmlModelMaturity.attrib["locked"] = False
        else:
            if setModelMaturity or ("updateExisting" in self.producerParameters and self.producerParameters["updateExisting"] is True):
                self.updator.initialize({COUNT: self.pmmlModelMaturity.attrib["numUpdates"]})
            self.lock = self.pmmlModelMaturity.attrib["locked"]

        self.consumerAlgorithm.initialize()
        self.producerAlgorithm.initialize(**self.producerParameters)

        self.constants = self.pmmlModel.child(pmml.Extension, exception=False)
        if self.constants is None:
            self.constants = NameSpaceReadOnly()
        else:
            self.constants = self.constants.child(pmml.X_ODG_CustomProcessingConstants, exception=False)
            if self.constants is None:
                self.constants = NameSpaceReadOnly()
            else:
                self.constants = self.constants.nameSpace

        self.userFriendly = getattr(self, "userFriendly", new.instance(Segment))
        self.userFriendly.name = self.name()
        self.userFriendly.pmmlPredicate = self.pmmlPredicate
        self.userFriendly.expression = self.expressionTree
        self.userFriendly.evaluate = self.predicateMatches
        self.userFriendly.pmmlModel = self.pmmlModel
        self.userFriendly.consumer = self.consumerAlgorithm
        self.userFriendly.producer = self.producerAlgorithm
        self.userFriendly.const = self.constants
        self.userFriendly.state = self.state

        if customProcessing is not None:
            db = customProcessing.persistentStorage.db
            if self.userFriendly.name not in db:
                db[self.userFriendly.name] = NameSpace()
            self.userFriendly.db = db[self.userFriendly.name]

    def state(self):
        """Return the current state (UNINITIALIZED -> IMMATURE ->
        MATURE -> LOCKED)."""

        if not hasattr(self, "lock"):
            return UNINITIALIZED
        elif self.lock:
            return LOCKED
        elif self.updator.count() >= self.maturityThreshold:
            return MATURE
        else:
            return IMMATURE

    def update(self, syncNumber):
        """Update the maturity counter.

        Only called (by Engine) if the producer algorithm was
        *successful*.
        """

        self.updator.increment(syncNumber, 0.)
        if self.lockingThreshold and self.updator.count() >= self.lockingThreshold:
            self.lock = True

        self.pmmlModelMaturity.attrib["numUpdates"] = self.updator.count()
        if self.lock:
            self.pmmlModelMaturity.attrib["locked"] = True

    def name(self):
        """Return the segment id (or auto-assigned segment id)."""

        return self.segmentNameRegistry.objectToName[id(self)]

    def __repr__(self):
        return "<SegmentRecord id=\"%s\" (%s) at 0x%02x>" % (self.name(), str(self.state()).lower(), id(self))

    def __del__(self):
        self.segmentNameRegistry.unregister(id(self))
Example #4
0
    def __init__(self, configuration, model=None, dataStream=None, rethrowExceptions=None):
        self.model = model
        self.dataStream = dataStream
        self.rethrowExceptions = rethrowExceptions
        self.fileNameOnException = None

        # get the configuration, in whatever form you find it
        if isinstance(configuration, config.AugustusConfiguration):
            pass
        elif isinstance(configuration, basestring):
            try:
                configuration = xmlbase.loadfile(configuration, config.Config, lineNumbers=True)
            except IOError:
                configuration = xmlbase.load(configuration, config.Config, lineNumbers=True)
        else:
            raise ConfigurationError("Configuration must be a pre-validated XML object, a fileName, or a literal configuration string.")
    
        # set up logging
        setupLogging(configuration.matches(lambda x: isinstance(x, (config.Logging, config.Metadata))))
        self.logger = logging.getLogger()
        self.metadata = logging.getLogger("metadata")

        # begin "initialization" phase
        for l in self.logger, self.metadata:
            if "initialization" in l.differentLevel:
                l.setLevel(l.differentLevel["initialization"])
            else:
                l.setLevel(l.naturalLevel)

        # get the model, in whatever form you find it
        self.logger.info("Loading PMML model.")
        self.metadata.startTiming("Time to load PMML model")
        modelFileName = "(none)"
        maturityThreshold = 0
        if self.model is None:
            modelInput = configuration.child(config.ModelInput, exception=None)
            if modelInput is None:
                raise ConfigurationError("If a model is not provided to MainLoop explicitly, it must be present in the configuration file.")

            fileLocation = modelInput["fileLocation"]
            if not fileLocation.startswith("http://") and not fileLocation.startswith("https://"):
                fileList = glob.glob(fileLocation)
                if len(fileList) > 1:
                    fileList = [f for f in fileList if self._modelExceptionIdentifier not in f]
                if len(fileList) == 0:
                    raise IOError("No files matched the ModelInput fileLocation \"%s\"." % fileLocation)

                selectmode = modelInput.attrib.get("selectmode", "lastAlphabetic")
                if selectmode == "mostRecent":
                    fileLocation = max(fileList, key=lambda x: os.stat(x).st_mtime)
                elif selectmode == "lastAlphabetic":
                    fileList.sort()
                    fileLocation = fileList[-1]
                else:
                    assert False

                if self._modelExceptionIdentifier in fileLocation:
                    self.logger.warning("Using a PMML model that was written on exception (fileName \"%s\")" % fileLocation)

            self.model = xmlbase.loadfile(fileLocation, pmml.X_ODG_PMML, lineNumbers=True)

            if "maturityThreshold" in modelInput.attrib: maturityThreshold = modelInput["maturityThreshold"]

        elif isinstance(self.model, pmml.PMML):
            pass
        elif isinstance(self.model, basestring):
            try:
                self.model, modelFileName = xmlbase.loadfile(self.model, pmml.X_ODG_PMML, lineNumbers=True), self.model
            except IOError:
                self.model = xmlbase.load(self.model, pmml.X_ODG_PMML, lineNumbers=True)
        else:
            raise ConfigurationError("Model must be a pre-validated XML object, a fileName, or a literal PMML string.")
        self.metadata.stopTiming("Time to load PMML model")
        self.metadata.data["PMML model file"] = modelFileName

        # globally set random number seeds
        if "randomSeed" in configuration.attrib:
            augustusRandomSeed = configuration["randomSeed"]
            random.seed(augustusRandomSeed)
            numpy.random.seed(augustusRandomSeed + 1)
        else:
            augustusRandomSeed = "unspecified"

        # globally set numpy error handling
        numpy.seterr(divide="raise", over="raise", under="ignore", invalid="raise")

        # update schemes (producerUpdateScheme may be redefined below)
        consumerUpdateScheme = self._getUpdateScheme(configuration.child(config.ConsumerBlending, exception=False))
        producerUpdateScheme = self._getUpdateScheme(None)

        # set up scoring output
        outputConfig = configuration.child(config.Output, exception=False)
        if outputConfig is None:
            self.outputWriter = None
        else:
            outputParams = {"pmmlFileName": modelFileName, "mode": outputConfig.destination.attrib.get("type", "XML").lower()}

            if isinstance(outputConfig.destination, config.ToFile):
                if outputConfig.destination.attrib.get("overwrite", False):
                    outputStream = codecs.open(outputConfig.destination["name"], "w", encoding="utf-8")
                else:
                    outputStream = codecs.open(outputConfig.destination["name"], "a", encoding="utf-8")
            elif isinstance(outputConfig.destination, config.ToStandardError):
                outputStream = sys.stderr
            elif isinstance(outputConfig.destination, config.ToStandardOut):
                outputStream = sys.stdout
            else:
                assert False

            reportTag = outputConfig.child("ReportTag", exception=False)
            if reportTag:
                outputParams["reportName"] = reportTag.attrib.get("name", "Report")

            eventTag = outputConfig.child("EventTag", exception=False)
            if eventTag:
                outputParams["eventName"] = eventTag.attrib.get("name", "Event")
                outputParams["pseudoEventName"] = eventTag.attrib.get("pseudoName", "pseudoEvent")

            self.outputWriter = OutputWriter(outputStream, **outputParams)

        # initialize for the case of no output model
        engineSettings = {"maturityThreshold": maturityThreshold, "augustusRandomSeed": augustusRandomSeed}
        self.modelWriter = None
        segmentationScheme = SegmentationScheme(None, self.model)
        self.updateFlag = False
        self.aggregateUpdateFlag = False

        producerAlgorithm = dict(config.producerAlgorithmDefaults)
        for pa in producerAlgorithm.values():
            validationResult = pa.validate()
            assert validationResult is None

        # set up output model, if present in the configuration
        modelSetup = configuration.child(config.ModelSetup, exception=False)
        engineSettings["hasProducer"] = modelSetup is not None
        if engineSettings["hasProducer"]:
            self.logger.info("Setting up model updating/producing.")

            producerBlending = modelSetup.child(config.ProducerBlending, exception=False)
            producerUpdateScheme = self._getUpdateScheme(producerBlending)
            if producerBlending is not None and producerBlending.contains(config.MaturityThreshold):
                maturityConfig = producerBlending.child(config.MaturityThreshold)
                engineSettings["maturityThreshold"] = int(maturityConfig.attrib.get("threshold", 1))
                try:
                    engineSettings["lockingThreshold"] = int(maturityConfig.attrib["lockingThreshold"])
                except KeyError:
                    engineSettings["lockingThreshold"] = None

            engineSettings["lockAllSegments"] = modelSetup.attrib.get("mode", None) == "lockExisting"
            if engineSettings["lockAllSegments"] and segmentationScheme is not None and not segmentationScheme._generic and not segmentationScheme._whiteList:
                self.logger.warning("The model is locked and no new segments are specified...new model files will be unchanged.")

            self.modelWriter = getModelWriter(modelSetup)
            if self.modelWriter is not None:
                if self.modelWriter.baseName is None:
                    self.fileNameOnException = self._modelExceptionIdentifier + ".pmml"
                else:
                    self.fileNameOnException = "".join([self.modelWriter.baseName, self._modelExceptionIdentifier, ".pmml"])
            else:
                self.logger.warning("There is no outputFile attribute in the ModelSetup; no new model file will be created.")

            segmentationScheme = SegmentationScheme(modelSetup.child(config.SegmentationSchema, exception=False), self.model)
            self.updateFlag = modelSetup.attrib.get("updateEvery", "event") in ("event", "both")
            self.aggregateUpdateFlag = modelSetup.attrib.get("updateEvery", "event") in ("aggregate", "both")

            for pa in modelSetup.matches(config.ProducerAlgorithm):
                producerAlgorithm[pa["model"]] = pa
            if modelSetup.attrib.get("mode", None) == "updateExisting":
                for pa in producerAlgorithm.values():
                    pa.parameters["updateExisting"] = True
            if modelSetup.attrib.get("mode", None) == "replaceExisting":
                for pa in producerAlgorithm.values():
                    pa.parameters["updateExisting"] = False

        # to score or not to score
        eventSettings = configuration.child(config.EventSettings, exception=False)
        if eventSettings is not None:
            self.logger.info("Setting up output.")
            self.scoreFlag = eventSettings["score"]
            self.outputFlag = eventSettings["output"]
        else:
            self.scoreFlag = False
            self.outputFlag = False

        aggregationConfig = configuration.child(config.AggregationSettings, exception=False)
        if aggregationConfig is not None:
            self.aggregateScoreFlag = aggregationConfig["score"]
            self.aggregateOutputFlag = aggregationConfig["output"]
            self.aggregationSettings = dict(aggregationConfig.attrib)
        else:
            self.aggregateScoreFlag = False
            self.aggregateOutputFlag = False
            self.aggregationSettings = None

        self.metadata.data["Update model"] = "true" if self.updateFlag or self.aggregateUpdateFlag else "false"

        # build a scoring engine once without a dataStream (to evaluate any verification blocks)
        self.engine = Engine(self.model, None, producerUpdateScheme, consumerUpdateScheme, segmentationScheme, producerAlgorithm, **engineSettings)
        self.engine.initialize()
        if self.outputWriter is not None: self.outputWriter.open()

        # begin "verification" phase
        for l in self.logger, self.metadata:
            if "verification" in l.differentLevel:
                l.eventLogLevel = l.differentLevel["verification"]
                l.setLevel(l.differentLevel["verification"])
            else:
                l.eventLogLevel = l.naturalLevel
                l.setLevel(l.naturalLevel)

        # evaluate verification blocks
        modelVerificationConfig = configuration.child(config.ModelVerification, exception=False)
        if modelVerificationConfig is not None:
            verify(modelVerificationConfig, self.engine, self.logger, self.outputWriter)

        # verification can increment aggregate variables, but
        # aggregates should all start at zero at the start of real
        # processing, whether verification happened or not
        self.engine.flushAggregates()

        # get the dataStream, in whatever form you find it
        self.logger.info("Setting up data input.")
        if self.dataStream is None:
            configDataInput = configuration.child(config.DataInput, exception=None)
            if configDataInput is None:
                raise ConfigurationError("If a dataStream is not provided to MainLoop explicitly, it must be present in the configuration file.")
            if configDataInput.contains(config.FromFile):
                self.dataStream = DataStreamer(configDataInput.child(config.FromFile), self.engine.pmmlModel)
            elif configDataInput.contains(config.FromStandardIn):
                self.dataStream = DataStreamer(configDataInput.child(config.FromStandardIn), self.engine.pmmlModel)
            elif configDataInput.contains(config.FromHTTP):
                self.dataStream = AugustusHTTPDataStream(configDataInput.child(config.FromHTTP))
                if self.outputWriter is None:
                    self.dataStream.respond = False
                if self.dataStream.respond:
                    self.dataStream.setupOutput(self.outputWriter)
            else:
                assert False

        # begin "eventLoop" phase
        for l in self.logger, self.metadata:
            if "eventloop" in l.differentLevel:
                l.eventLogLevel = l.differentLevel["eventloop"]
                l.setLevel(l.differentLevel["eventloop"])
            else:
                l.eventLogLevel = l.naturalLevel
                l.setLevel(l.naturalLevel)

        # possibly set up custom processing
        self.customProcessing = configuration.child(config.CustomProcessing, exception=False)
        if self.customProcessing is not None:
            constants = self.engine.pmmlModel.child(pmml.Extension, exception=False)
            if constants is None:
                constants = NameSpaceReadOnly()
            else:
                constants = constants.child(pmml.X_ODG_CustomProcessingConstants, exception=False)
                if constants is None:
                    constants = NameSpaceReadOnly()
                else:
                    constants = constants.nameSpace

            atoms = {"INVALID": INVALID, "MISSING": MISSING, "IMMATURE": IMMATURE, "MATURE": MATURE, "LOCKED": LOCKED, "UNINITIALIZED": UNINITIALIZED}
            for thing in pmml.OutputField.__dict__.values() + pmml.X_ODG_OutputField.__dict__.values():
                if isinstance(thing, Atom):
                    atoms[repr(thing)] = thing

            self.customProcessing.initialize(self.model, self.engine.pmmlModel, constants, [s.userFriendly for s in self.engine.segmentRecords], atoms, self.logger, self.metadata, consumerUpdateScheme, producerUpdateScheme)
            self.engine.customProcessing = self.customProcessing
            self.engine.reinitialize()

        else:
            # only turn off circular garbage collection if there is no CustomProcessing or AugustusInterface
            gc.disable()