Beispiel #1
0
    def __get_contexts(self, titles):

        contexts = []
        misses = []

        for title, context in self.__contexts_cache.get(titles).items():

            if context is not None:
                contexts.append(context)
            else:
                misses.append(title)

        if self.__on_miss_backoff is True:
            for title in misses:
                SQLiteDict.storage(
                    Config.get('CTI')['backed_off_search']
                    ['storage_name'])[title] = title

        else:

            new_contexts = self.__search_engine.contexts(misses)

            contexts += list(new_contexts.values())

            self.__contexts_cache.set_many(new_contexts)

        return contexts
Beispiel #2
0
    def test_post_record(self):
        topic: FlexioTopic = FlexioTopic()
        topic.title = 'test'
        topic.body = 'dudu'
        topic.state = IssueState.OPEN

        print(topic.to_api_dict())

        r: Response = FlexioClient(
            self.config_handler).post_record(record=topic)

        topic_created: FlexioTopic = FlexioTopic.build_from_api(r.json())
        self.assertEqual(topic.title, topic_created.title)
        self.assertEqual(topic.body, topic_created.body)
        self.assertEqual(topic.state, topic_created.state)

        print(topic_created.__dict__())

        self.assertIs(r.status_code, 200)
        falsy_config_handler = ConfigHandler(CONFIG_DIR)
        falsy_config_handler.config = Config().with_flexio(
            ConfigFlexio(activate=True, user_token='dudu'))
        r: Response = FlexioClient(falsy_config_handler).post_record(
            record=topic)
        self.assertIsNot(r.status_code, 200)
Beispiel #3
0
def start_server():

    signal.signal(signal.SIGINT, signal_exit_handler)

    backed_off_search_process = None

    if Config.get('CTI')['knowledge_base']['on_miss_backoff']:
        backed_off_search_process = Process(target=backed_off_search)
        backed_off_search_process.start()

    config = Config.get('server')

    server.run(host=config['host'], port=config['port'], threaded=True)

    if Config.get('CTI')['knowledge_base']['on_miss_backoff']:
        backed_off_search_process.join()
Beispiel #4
0
    def __init__(self, cache_name):

        self.__cache_name = cache_name

        if ES.connection('es') is None:
            Logger.log(__name__, 'could not connect to elasticsearch', type='error')
            return

        ESService.create_index(self.__cache_name, Config.get('elasticsearch')['indices_settings'][self.__cache_name])
Beispiel #5
0
    def test_get_users(self):
        r: Response = self.github_repo.get_users()
        self.assertIs(r.status_code, 200)
        print(r.json())

        falsy_config_handler = ConfigHandler(CONFIG_DIR)
        falsy_config_handler.config = Config().with_github(
            ConfigGithub(activate=True, user='******', token='dudu'))
        r: Response = Github(falsy_config_handler).get_user()
        self.assertIsNot(r.status_code, 200)
Beispiel #6
0
def clear_backedoff_storage():

    storage_path = Data.get(
        Config.get('CTI')['backed_off_search']['storage_name'])

    if os.path.isfile(storage_path):
        os.remove(storage_path)
        Logger.log(__name__, 'the backedoff search storage is now empty')
    else:
        Logger.log(__name__, 'the backedoff search storage is already empty')
Beispiel #7
0
    def __init__(self):

        self.__config = Config.get('CTI')['knowledge_base']

        self.__on_miss_backoff = self.__config['on_miss_backoff']

        self.__search_engine = SearchEngine(
            top_n_contexts_per_query=self.__config['max_contexts_per_query'])
        self.__query_cache = QueryCache.create(self.__config['query_cache'])
        self.__contexts_cache = ContextsCache.create(
            self.__config['contexts_cache'])
Beispiel #8
0
def handle_cti_request(terms, context):

    if len(terms) == 0:
        return {}

    scored = {}

    if (Config.get('CTI')['max_processes_per_job'] == 1) or (len(terms) == 1):

        for term in terms:
            scored[term] = round(cti.term_informativeness(term, context), 2)

    else:

        with Pool(Config.get('CTI')['max_processes_per_job']) as p:
            for term, score in p.map(
                    partial(cti_job, cti=cti, context=context), terms):
                scored[term] = score

    return scored
Beispiel #9
0
class Main(Game):
  def __init__(self, debug=False):
    Game.__init__(self)
    self.debug  = debug
    self.config = Config()
    self.event  = Event(self)
    
    # graphics must be created first because
    # some other modules need the HWND or D3DDEVICE
    self.graphics  = Graphics()
    self.io        = IO()
    self.input     = cpp.Input()
    self.sound     = Sound()
    self.physics   = Physics()
    
    # order that modules will be ticked in the main loop
    self.AddModule(self.io)
    self.AddModule(self.input)
    self.AddModule(self.sound)
    self.AddModule(self.physics)
    self.AddModule(self.graphics)
    if hasattr(cpp, 'TestModule'):
        self.test = cpp.TestModule();
    
    self.event.Register(self.QuitEvent)
    
  def Init(self):
    Game.Init(self)

  def Tick(self, time):
    # hack! we need the *current* physics results
    # to compute stable results
    self.physics.physics.GetPhysicsResults()
    
    self.states[-1].Tick(time)
    Game.Tick(self, time)
          
  def QuitEvent(self):
    self.state = 0
    self.config.save()
Beispiel #10
0
def bootstrap_knowledge_base(dump_directory):

    Logger.log(__name__, 'bootstrapping knowledge base...')

    contexts_cache = ContextsCache.create(
        Config.get('CTI')['knowledge_base']['contexts_cache'])

    for article in WikipediaUtils.get_articles(dump_directory,
                                               attributes=['title',
                                                           'context']):
        contexts_cache.set(article['title'], article['context'])
        Logger.log(__name__,
                   'bootstrapping got context for: ' + article['title'])
Beispiel #11
0
    def load_file_config(self) -> ConfigHandler:
        if not self.file_path().is_file():
            raise FileNotFoundError(
                self.file_path(),
                'Flexio Flow Core not initialized try : flexio-flow core config'
            )
        f: fileinput = self.file_path().open('r')
        # TODO: ensure 3.8 compatibility
        data: dict = yaml.load(f, Loader=yaml.FullLoader)
        # data: dict = yaml.load(f)
        f.close()

        self.__config = Config().with_github(
            github=ConfigGithub(
                activate=data.get('github', {}).get('activate', False),
                user=data.get('github', {}).get('user', ''),
                token=data.get('github', {}).get('token', '')
            )
        ).with_flexio(flexio=ConfigFlexio(
            activate=data.get('flexio', {}).get('activate', False),
            user_token=data.get('flexio', {}).get('user_token', '')
        )).with_branches_config(branches_config=BranchesConfig.from_dict(data.get('branches', {})))
        return self
Beispiel #12
0
def backed_off_search():

    config = Config.get('CTI')['backed_off_search']

    contexts_cache = ContextsCache.create(
        Config.get('CTI')['knowledge_base']['contexts_cache'])
    search_engine = SearchEngine()

    Logger.log(__name__, 'backed off search process started')
    Logger.log(
        __name__, 'backed off search storage has ' +
        str(len(SQLiteDict.storage(config['storage_name']))) + ' items')

    c = 0

    while True:

        try:
            title = SQLiteDict.storage(config['storage_name']).popitem()[0]
        except KeyError:
            sleep(config['empty_storage_wait_seconds'])
            continue

        contexts_cache.set(title, search_engine.context(title))

        Logger.log(__name__, 'backed off search got context for: ' + title)

        c += 1
        if c > 30:
            c = 0
            Logger.log(
                __name__, 'backed off search storage has ' +
                str(len(SQLiteDict.storage(config['storage_name']))) +
                ' items')

        sleep(config['seconds_between_searches'])
Beispiel #13
0
 def __init__(self, debug=False):
   Game.__init__(self)
   self.debug  = debug
   self.config = Config()
   self.event  = Event(self)
   
   # graphics must be created first because
   # some other modules need the HWND or D3DDEVICE
   self.graphics  = Graphics()
   self.io        = IO()
   self.input     = cpp.Input()
   self.sound     = Sound()
   self.physics   = Physics()
   
   # order that modules will be ticked in the main loop
   self.AddModule(self.io)
   self.AddModule(self.input)
   self.AddModule(self.sound)
   self.AddModule(self.physics)
   self.AddModule(self.graphics)
   if hasattr(cpp, 'TestModule'):
       self.test = cpp.TestModule();
   
   self.event.Register(self.QuitEvent)
 def setup_config_handler(cls) -> ConfigHandler:
     config_handler: ConfigHandler = ConfigHandler(cls.DIR_PATH_TEST)
     config_handler.config = Config().with_github(
         github=ConfigGithub(activate=True, user=USER, token=TOKEN_TEST))
     return config_handler
Beispiel #15
0
 def __setup(self):
     self.__conns = {}
     self.__configs = Config.get('elasticsearch')['connections']
Beispiel #16
0
 def setUp(self):
     self.config_handler = ConfigHandler(CONFIG_DIR)
     self.config_handler.config = Config().with_github(
         ConfigGithub(activate=True, user=USER, token=TOKEN_TEST))
     self.github_repo: Github = Github(self.config_handler).with_repo(
         Repo(owner='flexiooss', repo='flexio-flow-punching-ball'))
Beispiel #17
0
 def setUp(self):
     self.config_handler = ConfigHandler(CONFIG_DIR)
     self.config_handler.config = Config().with_flexio(
         ConfigFlexio(activate=True, user_token=USER_TOKEN))
Beispiel #18
0
 def reset_config(self) -> ConfigHandler:
     self.__config = Config()
     return self
Beispiel #19
0
class ConfigHandler:
    DEFAULT_FILE_NAME: str = 'config.yml'
    __config: Optional[Config]

    def __init__(self, dir_path: Path, filename: Optional[str] = None):
        self.dir_path: Path = dir_path
        self.__config: Optional[Config] = None
        self.__filename: str = ConfigHandler.DEFAULT_FILE_NAME if filename is None else filename

    @property
    def config(self) -> Config:
        return self.__config

    @config.setter
    def config(self, v: Config):
        self.__config = v

    def file_exists(self) -> bool:
        return self.file_path().is_file()

    def load_file_config(self) -> ConfigHandler:
        if not self.file_path().is_file():
            raise FileNotFoundError(
                self.file_path(),
                'Flexio Flow Core not initialized try : flexio-flow core config'
            )
        f: fileinput = self.file_path().open('r')
        # TODO: ensure 3.8 compatibility
        data: dict = yaml.load(f, Loader=yaml.FullLoader)
        # data: dict = yaml.load(f)
        f.close()

        self.__config = Config().with_github(
            github=ConfigGithub(
                activate=data.get('github', {}).get('activate', False),
                user=data.get('github', {}).get('user', ''),
                token=data.get('github', {}).get('token', '')
            )
        ).with_flexio(flexio=ConfigFlexio(
            activate=data.get('flexio', {}).get('activate', False),
            user_token=data.get('flexio', {}).get('user_token', '')
        )).with_branches_config(branches_config=BranchesConfig.from_dict(data.get('branches', {})))
        return self

    def write_file(self) -> str:
        stream = self.file_path().open('w')
        print(self.__config.to_dict())
        yaml.dump(self.__config.to_dict(), stream, default_flow_style=False)
        stream.close()
        print("""#################################################
Write file : {0!s} 
#################################################""".format(self.file_path().as_posix()))
        return yaml.dump(self.__config.to_dict(), default_flow_style=False)

    def file_path(self) -> Path:
        return self.dir_path / self.__filename

    def has_issuer(self) -> bool:
        return self.__config.github.activate is True

    def has_topicer(self) -> bool:
        return self.__config.flexio.activate is True

    def reset_config(self) -> ConfigHandler:
        self.__config = Config()
        return self

    def master(self) -> str:
        return self.config.branches_config.master

    def develop(self) -> str:
        return self.config.branches_config.develop

    def feature(self) -> str:
        return self.config.branches_config.feature

    def hotfix(self) -> str:
        return self.config.branches_config.hotfix

    def release(self) -> str:
        return self.config.branches_config.release
Beispiel #20
0
def clear_knowledge_base():
    ESService.delete_index(
        Config.get('CTI')['knowledge_base']['query_cache']['name'])
    ESService.delete_index(
        Config.get('CTI')['knowledge_base']['contexts_cache']['name'])
    Logger.log(__name__, 'the knowledge base is now empty')