def __init__(self, configDictionary):
super(MLBrain, self).__init__(configDictionary)
self.profile = {"name": "mlbrain-module", "class": "mlbrain"}
self.nlp_module = None
self.concept_module = None
self.data_storage_path = utils.getKeyFromSectionInConfiguration(
'brain', 'data_storage_path', None, configDictionary)
self.dictionary_data_source_path = utils.getKeyFromSectionInConfiguration(
'bender-training', 'dictionary_data_source_path', None,
configDictionary)
self.dictionary_output_path = utils.getKeyFromSectionInConfiguration(
'bender-training', 'dictionary_output_path', None,
configDictionary)
if self.dictionary_output_path == None:
print(
'**** ERROR: No Dictionary output path defined in bender-training section of config-file.'
)
sys.exit(1)
self.dict_filename = os.path.join(self.dictionary_output_path,
'dictionary.dict')
self.remove_stop_words = int(
utils.getKeyFromSectionInConfiguration('bender-training',
'remove_stop_words', '0',
configDictionary))
self.language = utils.getKeyFromSectionInConfiguration(
'bender-training', 'data_language_short', 'en', configDictionary)
self.stop_words = get_stop_words(self.language)
utils.safe_create_directory(self.dictionary_output_path)
if self.data_storage_path == None:
print('**** ERROR: No data storage path specified. Exiting!')
sys.exit(1)
self.dictionary_manager = GensimDictionary(
self.dictionary_data_source_path, self.dict_filename,
self.remove_stop_words, self.stop_words)
self._loadData()
def _prepareNNIQAFormat(self):
print("5/5: CONVERTING TRAINING DATA FOR NN in I-Q/A-FORMAT...")
questions = []
answers = {}
self.answers = self.brain.getAllAnswers()
self.questions = self.brain.getAllQAPairs()
for entry in self.answers:
answerText = entry['answer']
aID = int(entry['id'])
document = self.brain.getIndicesForText(answerText)
arr = []
for sentence in document:
arr.extend(sentence)
answers[aID] = arr
questions = []
for question in self.questions:
qText = question['question']
document = self.brain.getIndicesForText(qText)
arr = []
for sentence in document:
arr.extend(sentence)
questions.append({'question': arr, 'answers': question['answers']})
output_p = os.path.join(self.output_path, 'nn_iqa')
utils.safe_create_directory(output_p)
output_qt = os.path.join(output_p, 'questions.json')
output_qp = os.path.join(output_p, 'questions.pickle')
output_at = os.path.join(output_p, 'answers.json')
output_ap = os.path.join(output_p, 'answers.pickle')
json.dump(questions, open(output_qt, 'w'), indent=4)
json.dump(answers, open(output_at, 'w'), indent=4)
pickle.dump(questions, open(output_qp, 'wb'))
pickle.dump(answers, open(output_ap, 'wb'))
def __init__(self, configDictionary):
self.configDictionary = configDictionary
self.train_data_source_file = utils.getKeyFromSectionInConfiguration(
'bender-training', 'train_data_source_file', None,
configDictionary)
if not self.train_data_source_file:
print(
"Config does not contain 'train_data_source_file', please provide one."
)
exit(1)
self.query_media_type = utils.getKeyFromSectionInConfiguration(
'bender-training', 'query_media_type', None, configDictionary)
self.response_media_type = utils.getKeyFromSectionInConfiguration(
'bender-training', 'response_media_type', None, configDictionary)
self.raw_data_format = utils.getKeyFromSectionInConfiguration(
'bender-training', 'raw_data_format', None, configDictionary)
self.train_data_q_media_type = utils.getKeyFromSectionInConfiguration(
'bender-training', 'train_data_q_media_type', None,
configDictionary)
self.train_data_a_media_type = utils.getKeyFromSectionInConfiguration(
'bender-training', 'train_data_a_media_type', None,
configDictionary)
self.output_path = utils.getKeyFromSectionInConfiguration(
'bender-training', 'output_path', None, configDictionary)
self.train_data_queries_root_dir = utils.getKeyFromSectionInConfiguration(
'bender-training', 'converted_train_data_q_path', None,
configDictionary)
self.train_data_answers_dir = utils.getKeyFromSectionInConfiguration(
'bender-training', 'converted_train_data_a_path', None,
configDictionary)
self.generate_lsi = int(
utils.getKeyFromSectionInConfiguration('bender-training',
'generate_lsi', 0,
configDictionary))
concept = utils.getModulenameFromConfiguration(
'concept', 'modules.concept.mlconcept.MLConcept', configDictionary)
utils.validate_module_class(concept, MLConcept)
self.concept = utils.initialize_class(concept, configDictionary)
nlp = utils.getModulenameFromConfiguration('nlp',
'modules.nlp.mlnlp.MLNLP',
configDictionary)
utils.validate_module_class(nlp, MLNLP)
self.nlp = utils.initialize_class(nlp, configDictionary)
utils.safe_create_directory(self.output_path)
self.configDictionary = configDictionary
self.question_file = ''
self.answers_file = ''
self.questions = []
self.answers = []
self.internalMachineLogics = []
def _loadAnnoyIndex(self):
self.loading_lock.acquire()
self.dictionary = self.brain.getDictionary()
self.num_topics_lsi = int(
utils.getKeyFromSectionInConfiguration('bender-training',
'num_topics_lsi', 200,
self.config_dict))
self.output_path = self.module_config.get('annoy_data_path', '')
self.accuracy = int(self.module_config.get('accuracy', 500))
self.max_results = int(self.module_config.get('max_results', 100))
retraining_iv = int(
self.module_config.get('retraining_interval_in_minutes', 7))
self.retraining_interval_in_seconds = retraining_iv * 60
utils.safe_create_directory(self.output_path)
self.lsi_vectors_filename = os.path.join(self.output_path,
LSI_VECTORS_FNAME)
self.lsi_model_filename = os.path.join(self.output_path,
LSI_MODEL_FNAME)
self.tfidf_model_filename = os.path.join(self.output_path,
TFIDF_MODEL_FNAME)
self.annoy_index_filename = os.path.join(self.output_path,
ANNOY_OUTPUT_FNAME)
self.clipped_corpus_filename = os.path.join(self.output_path,
CLIPPED_CORPUS_FNAME)
if os.path.exists(self.annoy_index_filename):
self.mm = gensim.corpora.MmCorpus(self.lsi_vectors_filename)
num_features, num_docs = self.mm.num_terms, min(
self.mm.num_docs, MAX_DOCS)
self.index_annoy = annoy.AnnoyIndex(num_features, metric='angular')
self.index_annoy.load(self.annoy_index_filename)
else:
print(
'**** ERROR: Annoy index does not exist. Please train first!')
sys.exit(1)
if os.path.exists(self.lsi_model_filename):
self.lsi_model = gensim.models.LsiModel.load(
self.lsi_model_filename)
else:
print('**** ERROR: Annoy LSI Model missing. Please train first!')
sys.exit(1)
if os.path.exists(self.tfidf_model_filename):
self.tfidf_model = gensim.models.TfidfModel.load(
self.tfidf_model_filename)
self.loading_lock.release()
def __init__(self, configDictionary):
super(JSONStorage, self).__init__(configDictionary)
self.entries = {}
jsonDatabaseFile = utils.getKeyFromSectionInConfiguration(
'permanentstorage', 'json_storage_database',
'/tmp/-json-storage.json', configDictionary)
self.mutex_lock = threading.Lock()
self.jsonDataFile = jsonDatabaseFile
self.profile = {
"name": "json-storage",
"class": "permanent-storage",
"verson": "1.0"
}
if os.path.isfile(self.jsonDataFile):
self.entries = json.load(
codecs.open(self.jsonDataFile, 'r', 'utf-8'))
else:
utils.safe_create_directory(os.path.dirname(self.jsonDataFile))
def save_brain_data(data_storage_path, questions, answers, questions_indexed,
answers_indexed):
"""
This function is used to save the brain-data in a separate thread during Bender operations.
The problem otherwise would be that, depending on data size, the user has to wait for his
response quite some time (until save is finished).
The reason: Once BenderCore receives an answer from a HumanLogic, it tells the brain and
other modules to "learn" new data. After this learning, the brain needs to save itself to
hard disk, otherwise the data would get lost. Since this call happens in BenderCore just
in the "getResponse"-call, the return of the response to the REST-API would need to wait
until this save has been done.
Therefore, we do it in a separate thread.
It has actually no drawback as the newly learned data is still in RAM and can immediately
be used for next requests... (quite different from how it works in TextSimilarity and WMD)
"""
logger = logging.getLogger(os.path.basename(sys.argv[0]))
logger.info('Updating BRAIN DATA. Trying to get a mutex-lock...')
brain_mutex.acquire()
logger.info('Updating BRAIN DATA. successfully got a mutex-lock...')
try:
utils.safe_create_directory(data_storage_path)
data_file = os.path.join(data_storage_path, 'knowledge-raw.pickle')
knowledge = {'questions': questions, 'answers': answers}
pickle.dump(knowledge, open(data_file, 'wb'))
data_file = os.path.join(data_storage_path, 'knowledge-indexed.pickle')
knowledge = {
'questions': questions_indexed,
'answers': answers_indexed
}
pickle.dump(knowledge, open(data_file, 'wb'))
logger.info('Updating BRAIN DATA: Successfully updated brain-data...')
finally:
brain_mutex.release()
api.add_resource(APIHLIDeleteItem, API_ROOT + '/hli/delete')
api.add_resource(APIHLIUnlockLockItem, API_ROOT + '/hli/unlock')
api.add_resource(APIHLIAnswerItem, API_ROOT + '/hli/answer')
"""
If we're running as a standalone application (and cen respond to only one API-call
at the same time), run us in debug-mode with Flask's own server...
"""
if __name__ == '__main__':
api.add_resource(APISimpleHLI, '/ui/hli/')
api.add_resource(APISimpleUI, '/ui/')
app.add_url_rule('/ui/<path:file_relative_path_to_root>',
'serve_page',
serve_page,
methods=['GET'])
utils.safe_create_directory(os.path.dirname(accessLogFile))
logger = logging.getLogger('werkzeug')
logger.setLevel(logging.INFO)
handler = logging.FileHandler(accessLogFile)
logger.addHandler(handler)
logger.propagate = 0
app.logger.addHandler(handler)
print(
"========================================================================="
)
print(' | Starting web-server...')
print(' | HTTP-Logging to: ', accessLogFile, sep='')
print(' | Point your browser to : http://',
devServerIP,
':',
def _create_directories(self):
utils.safe_create_directory(self.output_root_path)
utils.safe_create_directory(self.output_root_path + '/corpus')
utils.safe_create_directory(self.output_root_path + '/models')
utils.safe_create_directory(self.output_root_path + '/dictionary')
utils.safe_create_directory(self.output_root_path + '/ids')
utils.safe_create_directory(self.output_root_path + '/vectors')
def create_annoy_index(moduleConfigSection, configDict, dictionary, corpus,
in_q, out_q):
logger = logging.getLogger(os.path.basename(sys.argv[0]))
module_config = moduleConfigSection
config_dict = configDict
num_topics_lsi = int(
utils.getKeyFromSectionInConfiguration('bender-training',
'num_topics_lsi', 200,
configDict))
output_path = module_config.get('annoy_data_path', '')
accuracy = int(module_config.get('accuracy', 500))
max_results = int(module_config.get('max_results', 100))
utils.safe_create_directory(output_path)
lsi_vectors_file = os.path.join(output_path, LSI_VECTORS_FNAME)
lsi_model_filename = os.path.join(output_path, LSI_MODEL_FNAME)
tfidf_vectors_file = os.path.join(output_path, TFIDF_MODEL_FNAME)
annoy_output_filename = os.path.join(output_path, ANNOY_OUTPUT_FNAME)
clipped_output_filename = os.path.join(output_path, CLIPPED_CORPUS_FNAME)
tfidf = gensim.models.TfidfModel(corpus)
logger.info('Saving Tfidf...')
tfidf.save(tfidf_vectors_file)
logger.info('*** START generating LSI...')
lsi = gensim.models.LsiModel(tfidf[corpus],
id2word=dictionary,
num_topics=num_topics_lsi)
logger.info('*** DONE generating LSI...')
lsi.save(lsi_model_filename)
logger.info('*** SAVED generating LSI...')
# convert all articles to latent semantic space, store the result as a MatrixMarket file
# normalize all vectors to unit length, to simulate cossim in libraries that only support euclidean distance
gensim.corpora.MmCorpus.serialize(lsi_vectors_file,
(gensim.matutils.unitvec(vec)
for vec in lsi[tfidf[corpus]]))
mm = gensim.corpora.MmCorpus(lsi_vectors_file)
num_features, num_docs = mm.num_terms, min(mm.num_docs, MAX_DOCS)
clipped = numpy.empty((num_docs, num_features), dtype=numpy.float32)
for docno, doc in enumerate(itertools.islice(mm, num_docs)):
clipped[docno] = gensim.matutils.sparse2full(doc, num_features)
logger.info('*** Saving clipped corpus as NUMPY...')
numpy.save(clipped_output_filename, clipped)
logger.info('*** Generating ANNOY...')
clipped_corpus = gensim.matutils.Dense2Corpus(clipped,
documents_columns=False)
index_annoy = annoy.AnnoyIndex(num_features, metric='angular')
for i, vec in enumerate(clipped_corpus):
index_annoy.add_item(
i,
list(gensim.matutils.sparse2full(vec, num_features).astype(float)))
logger.info('*** Building ANNOY...')
index_annoy.build(accuracy)
logger.info('*** Saving ANNOY...')
index_annoy.save(annoy_output_filename)
out_q.put('DONE')
out_q.close()
done = in_q.get()
os._exit(0)
return True
def _generate_word2vec(self):
global WMD_FILE
print(' | Generating Word2Vec model...')
wmd_data_path = self.module_config.get('wmd_data_path', None)
if wmd_data_path == None:
logger.info(
'**** Error: no output_root_path specified in section my i-machinelogic-section in the config-file.'
)
sys.exit(1)
else:
utils.safe_create_directory(wmd_data_path)
w2v_dim = 100
w2v_mc = 5
w2v_win = 5
w2v_iter = 10
w2v_workers = 4
create_nn_embed = 0
nn_embed_path = None
if self.config_dict is not None:
w2v_dim = int(self.config_dict['bender-training'].get(
'word2vec_dims', w2v_dim))
w2v_mc = int(self.config_dict['bender-training'].get(
'word2vec_min_count', w2v_mc))
w2v_win = int(self.config_dict['bender-training'].get(
'word2vec_window', w2v_win))
w2v_iter = int(self.config_dict['bender-training'].get(
'word2vec_iter', w2v_iter))
w2v_workers = int(self.config_dict['bender-training'].get(
'word2vec_workers', w2v_workers))
create_nn_embed = int(self.config_dict['bender-training'].get(
'create_nn_embed', 0))
nn_embed_path = self.config_dict['bender-training'].get(
'nn_embed_path', None)
train_data_source_dir = self.config_dict['bender-training'].get(
'train_data_source_file', None)
remove_stop_words = int(self.config_dict['bender-training'].get(
'remove_stop_words', 0))
language = self.config_dict['bender-training'].get(
'data_language_short', 'en')
stop_words = get_stop_words(language)
if train_data_source_dir is not None:
train_data_source_dir = os.path.dirname(train_data_source_dir)
dict_source_dir = self.config_dict['bender-training'].get(
'dictionary_data_source_path')
else:
print('Error: SOMETHING IS REALLY BAD HERE. No CONFIG-FILE FOUND')
sys.exit(1)
if train_data_source_dir is not None:
sentences = read_all_sentences(train_data_source_dir,
remove_stop_words, stop_words)
else:
sentences = []
sentences += read_all_sentences(dict_source_dir, remove_stop_words,
stop_words)
print(' | ... starting word2vec generation ...')
model = models.Word2Vec(sentences,
size=w2v_dim,
min_count=w2v_mc,
iter=w2v_iter,
window=w2v_win,
workers=w2v_workers,
sg=1)
wmd_w2v_filename = os.path.join(wmd_data_path, WMD_FILE)
model.save(wmd_w2v_filename)
if create_nn_embed == 1 and nn_embed_path is not None:
weights = model.wv.syn0
d = dict([(k, v.index) for k, v in model.wv.vocab.items()])
emb = numpy.zeros(shape=(len(self.doc2id) + 1, w2v_dim),
dtype='float32')
for i, w in self.doc2id.items():
if w not in d: continue
emb[i, :] = weights[d[w], :]
numpy.save(
open(os.path.join(nn_embed_path, 'word2vec_nn.embed'), 'wb'),
emb)
print(' | ... done')
def __init__(self, configDictionary):
self.configuration = configDictionary
self.security = BenderSecurity()
logFile = utils.getKeyFromSectionInConfiguration('bender-core', 'bender_core_logfile', 'logs/bender-core.log', configDictionary)
utils.safe_create_directory(os.path.dirname(logFile))
logging.basicConfig(level=logging.WARNING, filename=logFile + '.libs.log', format=LOG_FORMAT)
self.logger = logging.getLogger(os.path.basename(sys.argv[0]))
loggingFileHandler = logging.FileHandler(logFile)
loggingFileHandler.setLevel(logging.INFO)
loggingMemoryHandler = logging.handlers.MemoryHandler(128, target=loggingFileHandler)
loggingMemoryHandler.setLevel(logging.INFO)
loggingFormatter = logging.Formatter(LOG_FORMAT)
self.logger.addHandler(loggingMemoryHandler)
loggingFileHandler.setFormatter(loggingFormatter)
loggingMemoryHandler.setFormatter(loggingFormatter)
self.logger.setLevel(logging.INFO)
self.logger.info('###################### STARTING A NEW BENDER INSTANCE #######################')
self.logger.info('running %s' % ' '.join(sys.argv))
self.interactive = int(utils.getKeyFromSectionInConfiguration('bender-core', 'interactive', 0, configDictionary))
self.num_results = int(utils.getKeyFromSectionInConfiguration('bender-core', 'num_results', 1, configDictionary))
self.use_hli = int(utils.getKeyFromSectionInConfiguration('bender-core', 'use_hli', 1, configDictionary))
self.use_lookup = int(utils.getKeyFromSectionInConfiguration('bender-core', 'use_lookup', 1, configDictionary))
self.name = utils.getKeyFromSectionInConfiguration('bender-core', 'name', 'Bender', configDictionary)
self.personality = utils.getKeyFromSectionInConfiguration('bender-core', 'personality', 'mail-responder', configDictionary)
self.lowerCL = float(utils.getKeyFromSectionInConfiguration('bender-core', 'lowerconfidence_level', 0.85, configDictionary))
self.higherCL = float(utils.getKeyFromSectionInConfiguration('bender-core', 'higherconfidence_level', 0.95, configDictionary))
self.reinforcementTimeout = int(utils.getKeyFromSectionInConfiguration('bender-core', 'reinforcement_timeout', 7 * 24 * 60 * 60, configDictionary))
self.useSimilarity = int(utils.getKeyFromSectionInConfiguration('bender-core', 'use_similarity', 1, configDictionary))
transient_storage = utils.getModulenameFromConfiguration('transientstorage', 'modules.storage.transientstorage.MLTransientStorage', configDictionary)
permanent_storage = utils.getModulenameFromConfiguration('permanentstorage', 'modules.storage.permanentstorage.MLPermanentStorage', configDictionary)
lookup = utils.getModulenameFromConfiguration('lookup', 'modules.lookup.mllookup.MLLookup', configDictionary)
session = utils.getModulenameFromConfiguration('session', 'modules.session.mlsession.MLSession', configDictionary)
nlp = utils.getModulenameFromConfiguration('nlp', 'modules.nlp.mlnlp.MLNLP', configDictionary)
humanLogic = utils.getModulenameFromConfiguration('humanlogic', 'modules.humanlogic.mlhumanlogic.MLHumanLogic', configDictionary)
concept = utils.getModulenameFromConfiguration('concept', 'modules.concept.mlconcept.MLConcept', configDictionary)
stt = utils.getModulenameFromConfiguration('stt', 'modules.speech.mlstt.MLSTT', configDictionary)
tts = utils.getModulenameFromConfiguration('tts', 'modules.speech.mltts.MLTTS', configDictionary)
responseProcessor = utils.getModulenameFromConfiguration('response-postprocessor', 'modules.response.mlresponseproc.MLResponseProcessor', configDictionary)
dataInfusor = utils.getModulenameFromConfiguration('datainfusor', 'modules.datainfusor.mldatainfusor.MLDataInfusor', configDictionary)
similarity = utils.getModulenameFromConfiguration('similarity', 'module.similarity.mlsimilarity.MLSimilarity', configDictionary)
indexedResponseProc = utils.getModulenameFromConfiguration('indexed-response-processor', 'module.response.mlidxresponseproc.MLIndexedResponseProcessor', configDictionary)
brain = utils.getModulenameFromConfiguration('brain', 'modules.brain.mlbrain.MLBrain', configDictionary)
spelling = utils.getModulenameFromConfiguration('spelling', 'modules.spelling.mlspelling.MLSpelling', configDictionary)
utils.validate_module_class(transient_storage, MLTransientStorage)
self.transientStorage = utils.initialize_class(transient_storage, configDictionary)
utils.validate_module_class(permanent_storage, MLPermanentStorage)
self.permanentStorage = utils.initialize_class(permanent_storage, configDictionary)
utils.validate_module_class(lookup, MLLookup)
self.lookup = utils.initialize_class(lookup, configDictionary)
utils.validate_module_class(session, MLSession)
self.session = utils.initialize_class(session, configDictionary)
utils.validate_module_class(nlp, MLNLP)
self.nlp = utils.initialize_class(nlp, configDictionary)
utils.validate_module_class(humanLogic, MLHumanLogic)
self.humanLogic = utils.initialize_class(humanLogic, configDictionary)
self.humanLogicClass = humanLogic
utils.validate_module_class(concept, MLConcept)
self.concept = utils.initialize_class(concept, configDictionary)
utils.validate_module_class(stt, MLSpeechToText)
self.stt = utils.initialize_class(stt, configDictionary)
utils.validate_module_class(tts, MLTextToSpeech)
self.tts = utils.initialize_class(tts, configDictionary)
utils.validate_module_class(dataInfusor, MLDataInfusor)
self.dataInfusor = utils.initialize_class(dataInfusor, configDictionary)
utils.validate_module_class(responseProcessor, MLResponseProcessor)
self.responseProcessor = utils.initialize_class(responseProcessor, configDictionary)
utils.validate_module_class(similarity, MLSimilarity)
self.similarity = utils.initialize_class(similarity, configDictionary)
utils.validate_module_class(indexedResponseProc, MLIndexedResponseProcessor)
self.indexedResponseProcessor = utils.initialize_class(indexedResponseProc, configDictionary)
utils.validate_module_class(brain, MLBrain)
self.brain = utils.initialize_class(brain, configDictionary)
utils.validate_module_class(spelling, MLSpelling)
self.spelling = utils.initialize_class(spelling, configDictionary)
self.machineLogic = MLMachineLogic(configDictionary)
# NOTE: Even though we try to instantiate the classes in the right order
# and try to call their 'initForBender' in the right order, you should NEVER, in your module
# depend on any of the modules being already initialized in your implementation of these two
# methods:
# __init__
# initForBender
# Instead, you should request any other module instance only when you actually need them
# during the processing of data
#
# The only exception is MLBrain that relies on concept and nlp being already initialized before
# itself is called ... and this only during training...
self.permanentStorage.initForBender(self)
self.transientStorage.initForBender(self)
self.nlp.initForBender(self)
self.concept.initForBender(self)
self.brain.initForBender(self)
self.lookup.initForBender(self)
self.session.initForBender(self)
self.humanLogic.initForBender(self)
self.stt.initForBender(self)
self.tts.initForBender(self)
self.dataInfusor.initForBender(self)
self.responseProcessor.initForBender(self)
self.similarity.initForBender(self)
self.indexedResponseProcessor.initForBender(self)
self.machineLogic.initForBender(self)
self.spelling.initForBender(self)
self.dataProviders = []
dataproviders = utils.getSectionFromConfiguration('dataproviders', [ 'modules.dataproviders.mldataprovider.MLDataProvier' ], configDictionary)
for dp in dataproviders:
module = dp['module']
utils.validate_module_class(module, MLDataProvider)
newInstance = utils.initialize_class_with_config_section(module, dp, configDictionary)
newInstance.initForBender(self)
self.dataProviders.append(newInstance)
self.dataExtractors = []
dataextractors = utils.getSectionFromConfiguration('dataextractors', [ 'modules.dataextractors.mldataextrator.MLDataExtractor' ], configDictionary)
for de in dataextractors:
module = de['module']
utils.validate_module_class(module, MLDataExtractor)
newInstance = utils.initialize_class_with_config_section(module, de, configDictionary)
newInstance.initForBender(self)
self.dataExtractors.append(newInstance)
self.internalMachineLogics = []
internalMLs = utils.getSectionFromConfiguration('i-machinelogic', None, configDictionary)
for iml in internalMLs:
module = iml['module']
utils.validate_module_class(module, MLInternalMachineLogic)
newInstance = utils.initialize_class_with_config_section(module, iml, configDictionary)
newInstance.initForBender(self)
self.internalMachineLogics.append(newInstance)
self.preprocessors = []
preprocs = utils.getSectionFromConfiguration('preprocessors', None, configDictionary)
for preproc in preprocs:
module = preproc['module']
utils.validate_module_class(module, MLRequestProcessor)
newInstance = utils.initialize_class_with_config_section(module, preproc, configDictionary)
newInstance.initForBender(self)
self.preprocessors.append(newInstance)
self.jobData = {}
self.sessionData = {}
self.configuration = configDictionary
self.dataInfusor.setDataExtractorsAndProviders(self.dataExtractors, self.dataProviders)
self.benderjob_logger = BenderJobLogger(configDictionary)