def __get_contexts(self, titles):
contexts = []
misses = []
for title, context in self.__contexts_cache.get(titles).items():
if context is not None:
contexts.append(context)
else:
misses.append(title)
if self.__on_miss_backoff is True:
for title in misses:
SQLiteDict.storage(
Config.get('CTI')['backed_off_search']
['storage_name'])[title] = title
else:
new_contexts = self.__search_engine.contexts(misses)
contexts += list(new_contexts.values())
self.__contexts_cache.set_many(new_contexts)
return contexts
def test_post_record(self):
topic: FlexioTopic = FlexioTopic()
topic.title = 'test'
topic.body = 'dudu'
topic.state = IssueState.OPEN
print(topic.to_api_dict())
r: Response = FlexioClient(
self.config_handler).post_record(record=topic)
topic_created: FlexioTopic = FlexioTopic.build_from_api(r.json())
self.assertEqual(topic.title, topic_created.title)
self.assertEqual(topic.body, topic_created.body)
self.assertEqual(topic.state, topic_created.state)
print(topic_created.__dict__())
self.assertIs(r.status_code, 200)
falsy_config_handler = ConfigHandler(CONFIG_DIR)
falsy_config_handler.config = Config().with_flexio(
ConfigFlexio(activate=True, user_token='dudu'))
r: Response = FlexioClient(falsy_config_handler).post_record(
record=topic)
self.assertIsNot(r.status_code, 200)
def start_server():
signal.signal(signal.SIGINT, signal_exit_handler)
backed_off_search_process = None
if Config.get('CTI')['knowledge_base']['on_miss_backoff']:
backed_off_search_process = Process(target=backed_off_search)
backed_off_search_process.start()
config = Config.get('server')
server.run(host=config['host'], port=config['port'], threaded=True)
if Config.get('CTI')['knowledge_base']['on_miss_backoff']:
backed_off_search_process.join()
def __init__(self, cache_name):
self.__cache_name = cache_name
if ES.connection('es') is None:
Logger.log(__name__, 'could not connect to elasticsearch', type='error')
return
ESService.create_index(self.__cache_name, Config.get('elasticsearch')['indices_settings'][self.__cache_name])
def test_get_users(self):
r: Response = self.github_repo.get_users()
self.assertIs(r.status_code, 200)
print(r.json())
falsy_config_handler = ConfigHandler(CONFIG_DIR)
falsy_config_handler.config = Config().with_github(
ConfigGithub(activate=True, user='******', token='dudu'))
r: Response = Github(falsy_config_handler).get_user()
self.assertIsNot(r.status_code, 200)
def clear_backedoff_storage():
storage_path = Data.get(
Config.get('CTI')['backed_off_search']['storage_name'])
if os.path.isfile(storage_path):
os.remove(storage_path)
Logger.log(__name__, 'the backedoff search storage is now empty')
else:
Logger.log(__name__, 'the backedoff search storage is already empty')
def __init__(self):
self.__config = Config.get('CTI')['knowledge_base']
self.__on_miss_backoff = self.__config['on_miss_backoff']
self.__search_engine = SearchEngine(
top_n_contexts_per_query=self.__config['max_contexts_per_query'])
self.__query_cache = QueryCache.create(self.__config['query_cache'])
self.__contexts_cache = ContextsCache.create(
self.__config['contexts_cache'])
def handle_cti_request(terms, context):
if len(terms) == 0:
return {}
scored = {}
if (Config.get('CTI')['max_processes_per_job'] == 1) or (len(terms) == 1):
for term in terms:
scored[term] = round(cti.term_informativeness(term, context), 2)
else:
with Pool(Config.get('CTI')['max_processes_per_job']) as p:
for term, score in p.map(
partial(cti_job, cti=cti, context=context), terms):
scored[term] = score
return scored
class Main(Game):
def __init__(self, debug=False):
Game.__init__(self)
self.debug = debug
self.config = Config()
self.event = Event(self)
# graphics must be created first because
# some other modules need the HWND or D3DDEVICE
self.graphics = Graphics()
self.io = IO()
self.input = cpp.Input()
self.sound = Sound()
self.physics = Physics()
# order that modules will be ticked in the main loop
self.AddModule(self.io)
self.AddModule(self.input)
self.AddModule(self.sound)
self.AddModule(self.physics)
self.AddModule(self.graphics)
if hasattr(cpp, 'TestModule'):
self.test = cpp.TestModule();
self.event.Register(self.QuitEvent)
def Init(self):
Game.Init(self)
def Tick(self, time):
# hack! we need the *current* physics results
# to compute stable results
self.physics.physics.GetPhysicsResults()
self.states[-1].Tick(time)
Game.Tick(self, time)
def QuitEvent(self):
self.state = 0
self.config.save()
def bootstrap_knowledge_base(dump_directory):
Logger.log(__name__, 'bootstrapping knowledge base...')
contexts_cache = ContextsCache.create(
Config.get('CTI')['knowledge_base']['contexts_cache'])
for article in WikipediaUtils.get_articles(dump_directory,
attributes=['title',
'context']):
contexts_cache.set(article['title'], article['context'])
Logger.log(__name__,
'bootstrapping got context for: ' + article['title'])
def load_file_config(self) -> ConfigHandler:
if not self.file_path().is_file():
raise FileNotFoundError(
self.file_path(),
'Flexio Flow Core not initialized try : flexio-flow core config'
)
f: fileinput = self.file_path().open('r')
# TODO: ensure 3.8 compatibility
data: dict = yaml.load(f, Loader=yaml.FullLoader)
# data: dict = yaml.load(f)
f.close()
self.__config = Config().with_github(
github=ConfigGithub(
activate=data.get('github', {}).get('activate', False),
user=data.get('github', {}).get('user', ''),
token=data.get('github', {}).get('token', '')
)
).with_flexio(flexio=ConfigFlexio(
activate=data.get('flexio', {}).get('activate', False),
user_token=data.get('flexio', {}).get('user_token', '')
)).with_branches_config(branches_config=BranchesConfig.from_dict(data.get('branches', {})))
return self
def backed_off_search():
config = Config.get('CTI')['backed_off_search']
contexts_cache = ContextsCache.create(
Config.get('CTI')['knowledge_base']['contexts_cache'])
search_engine = SearchEngine()
Logger.log(__name__, 'backed off search process started')
Logger.log(
__name__, 'backed off search storage has ' +
str(len(SQLiteDict.storage(config['storage_name']))) + ' items')
c = 0
while True:
try:
title = SQLiteDict.storage(config['storage_name']).popitem()[0]
except KeyError:
sleep(config['empty_storage_wait_seconds'])
continue
contexts_cache.set(title, search_engine.context(title))
Logger.log(__name__, 'backed off search got context for: ' + title)
c += 1
if c > 30:
c = 0
Logger.log(
__name__, 'backed off search storage has ' +
str(len(SQLiteDict.storage(config['storage_name']))) +
' items')
sleep(config['seconds_between_searches'])
def __init__(self, debug=False):
Game.__init__(self)
self.debug = debug
self.config = Config()
self.event = Event(self)
# graphics must be created first because
# some other modules need the HWND or D3DDEVICE
self.graphics = Graphics()
self.io = IO()
self.input = cpp.Input()
self.sound = Sound()
self.physics = Physics()
# order that modules will be ticked in the main loop
self.AddModule(self.io)
self.AddModule(self.input)
self.AddModule(self.sound)
self.AddModule(self.physics)
self.AddModule(self.graphics)
if hasattr(cpp, 'TestModule'):
self.test = cpp.TestModule();
self.event.Register(self.QuitEvent)
def setup_config_handler(cls) -> ConfigHandler:
config_handler: ConfigHandler = ConfigHandler(cls.DIR_PATH_TEST)
config_handler.config = Config().with_github(
github=ConfigGithub(activate=True, user=USER, token=TOKEN_TEST))
return config_handler
def __setup(self):
self.__conns = {}
self.__configs = Config.get('elasticsearch')['connections']
def setUp(self):
self.config_handler = ConfigHandler(CONFIG_DIR)
self.config_handler.config = Config().with_github(
ConfigGithub(activate=True, user=USER, token=TOKEN_TEST))
self.github_repo: Github = Github(self.config_handler).with_repo(
Repo(owner='flexiooss', repo='flexio-flow-punching-ball'))
def setUp(self):
self.config_handler = ConfigHandler(CONFIG_DIR)
self.config_handler.config = Config().with_flexio(
ConfigFlexio(activate=True, user_token=USER_TOKEN))
def reset_config(self) -> ConfigHandler:
self.__config = Config()
return self
class ConfigHandler:
DEFAULT_FILE_NAME: str = 'config.yml'
__config: Optional[Config]
def __init__(self, dir_path: Path, filename: Optional[str] = None):
self.dir_path: Path = dir_path
self.__config: Optional[Config] = None
self.__filename: str = ConfigHandler.DEFAULT_FILE_NAME if filename is None else filename
@property
def config(self) -> Config:
return self.__config
@config.setter
def config(self, v: Config):
self.__config = v
def file_exists(self) -> bool:
return self.file_path().is_file()
def load_file_config(self) -> ConfigHandler:
if not self.file_path().is_file():
raise FileNotFoundError(
self.file_path(),
'Flexio Flow Core not initialized try : flexio-flow core config'
)
f: fileinput = self.file_path().open('r')
# TODO: ensure 3.8 compatibility
data: dict = yaml.load(f, Loader=yaml.FullLoader)
# data: dict = yaml.load(f)
f.close()
self.__config = Config().with_github(
github=ConfigGithub(
activate=data.get('github', {}).get('activate', False),
user=data.get('github', {}).get('user', ''),
token=data.get('github', {}).get('token', '')
)
).with_flexio(flexio=ConfigFlexio(
activate=data.get('flexio', {}).get('activate', False),
user_token=data.get('flexio', {}).get('user_token', '')
)).with_branches_config(branches_config=BranchesConfig.from_dict(data.get('branches', {})))
return self
def write_file(self) -> str:
stream = self.file_path().open('w')
print(self.__config.to_dict())
yaml.dump(self.__config.to_dict(), stream, default_flow_style=False)
stream.close()
print("""#################################################
Write file : {0!s}
#################################################""".format(self.file_path().as_posix()))
return yaml.dump(self.__config.to_dict(), default_flow_style=False)
def file_path(self) -> Path:
return self.dir_path / self.__filename
def has_issuer(self) -> bool:
return self.__config.github.activate is True
def has_topicer(self) -> bool:
return self.__config.flexio.activate is True
def reset_config(self) -> ConfigHandler:
self.__config = Config()
return self
def master(self) -> str:
return self.config.branches_config.master
def develop(self) -> str:
return self.config.branches_config.develop
def feature(self) -> str:
return self.config.branches_config.feature
def hotfix(self) -> str:
return self.config.branches_config.hotfix
def release(self) -> str:
return self.config.branches_config.release
def clear_knowledge_base():
ESService.delete_index(
Config.get('CTI')['knowledge_base']['query_cache']['name'])
ESService.delete_index(
Config.get('CTI')['knowledge_base']['contexts_cache']['name'])
Logger.log(__name__, 'the knowledge base is now empty')