def compute(self, item_id): # refresh Tracked term if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type( 'word'): self.list_tracked_words = Term.get_tracked_words_list() self.last_refresh_word = time.time() self.redis_logger.debug('Tracked word refreshed') print('Tracked word refreshed') if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type( 'set'): self.set_tracked_words_list = Term.get_set_tracked_words_list() self.last_refresh_set = time.time() self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') # Cast message as Item item = Item(item_id) item_date = item.get_date() item_content = item.get_content() signal.alarm(self.max_execution_time) dict_words_freq = None try: dict_words_freq = Term.get_text_word_frequency(item_content) except TimeoutException: self.redis_logger.warning(f"{item.get_id()} processing timeout") else: signal.alarm(0) if dict_words_freq: # create token statistics # for word in dict_words_freq: # Term.create_token_statistics(item_date, word, dict_words_freq[word]) item_source = item.get_source() # check solo words ####### # TODO: check if source needed ####### for word in self.list_tracked_words: if word in dict_words_freq: self.new_term_found(word, 'word', item.get_id(), item_source) # check words set for elem in self.set_tracked_words_list: list_words = elem[0] nb_words_threshold = elem[1] word_set = elem[2] nb_uniq_word = 0 for word in list_words: if word in dict_words_freq: nb_uniq_word += 1 if nb_uniq_word >= nb_words_threshold: self.new_term_found(word_set, 'set', item.get_id(), item_source)
def __init__(self): super(Tracker_Term, self).__init__() self.pending_seconds = 5 self.max_execution_time = self.process.config.getint( 'Tracker_Term', "max_execution_time") self.full_item_url = self.process.config.get( "Notifications", "ail_domain") + "/object/item?id=" # loads tracked words self.list_tracked_words = Term.get_tracked_words_list() self.last_refresh_word = time.time() self.set_tracked_words_list = Term.get_set_tracked_words_list() self.last_refresh_set = time.time() self.redis_logger.info(f"Module: {self.module_name} Launched")
def __init__(self): super(TermTrackerMod, self).__init__() self.pending_seconds = 5 self.max_execution_time = self.process.config.getint( 'TermTrackerMod', "max_execution_time") self.full_item_url = self.process.config.get( "Notifications", "ail_domain") + "/object/item?id=" # loads tracked words self.list_tracked_words = Term.get_tracked_words_list() self.last_refresh_word = time.time() self.set_tracked_words_list = Term.get_set_tracked_words_list() self.last_refresh_set = time.time() # Send module state to logs self.redis_logger.info("Module %s initialized" % (self._module_name()))
import NotificationHelper from packages import Item from packages import Term sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules')) import Flask_config full_item_url = "/showsavedpaste/?paste=" mail_body_template = "AIL Framework,\nNew occurrence for term tracked term: {}\nitem id: {}\nurl: {}{}" # loads tracked words list_tracked_words = Term.get_tracked_words_list() last_refresh_word = time.time() set_tracked_words_list = Term.get_set_tracked_words_list() last_refresh_set = time.time() class TimeoutException(Exception): pass def timeout_handler(signum, frame): raise TimeoutException signal.signal(signal.SIGALRM, timeout_handler) def new_term_found(term, term_type, item_id, item_date):