def __init__(self, debug=False): if not SceneParser.matcher: SceneParser.matcher = Matcher(PATTERN_GROUPS) Logr.info("Fragment matcher for %s created", self.__class__.__name__) super(SceneParser, self).__init__(SceneParser.matcher, debug)
def get_usage(self, month=None, year=None): if not month: month = datetime.date.today().month if not year: year = datetime.date.today().year # Check if we have already cached the result obj_hash = self.cache_hash(month, year) if self.cache_valid(obj_hash): return self.cache[obj_hash]['result'] # Ensure we are logged in if not self.conn: self.login() result = None try: Logr.info("Retrieving latest data usage details") usage = self.conn.get_usage(month, year) result = { 'total': usage.total, 'days': usage.days } except CompassConnectionException: pass # Cache the result self.cache[obj_hash] = { 'result': result, 'date': datetime.datetime.utcnow() } return result
def load_directory(self, sub_dir, base_type): # Load directory module first m = __import__(sub_dir) for module_name in getattr(self, sub_dir): Logr.info("Loading '%s.%s'" % (sub_dir, module_name)) __import__(sub_dir + '.' + module_name) module = getattr(m, module_name) # Find classes that extend the 'base_type' base class module_class = None for name, obj in inspect.getmembers(module, inspect.isclass): if obj is not base_type and issubclass(obj, base_type): if module_class is None: module_class = obj else: Logr.warning("Only one class is allowed per module") if module_class is not None: spec = { 'module_name': sub_dir + '.' + module_name, 'module': module, 'class': module_class, 'detail': module_class.__detail__, 'options': module_class.__options__ } getattr(self, sub_dir)[module_name] = spec self.modules[sub_dir + '.' + module_name] = spec else: Logr.warning("Unable to find class inside module '%s.%s'" % (sub_dir, module_name))
def construct_patterns(self, pattern_groups): compile_start = datetime.now() compile_count = 0 for group_name, patterns in pattern_groups: if group_name not in self.regex: self.regex[group_name] = [] # Transform into weight groups if type(patterns[0]) is str or type( patterns[0][0]) not in [int, float]: patterns = [(1.0, patterns)] for weight, patterns in patterns: weight_patterns = [] for pattern in patterns: # Transform into multi-fragment patterns if type(pattern) is str: pattern = (pattern, ) if type(pattern) is tuple and len(pattern) == 2: if type(pattern[0]) is str and is_list_type( pattern[1], str): pattern = (pattern, ) result = [] for value in pattern: if type(value) is tuple: if len(value) == 2: # Construct OR-list pattern value = value[0] % '|'.join(value[1]) elif len(value) == 1: value = value[0] result.append(re.compile(value, re.IGNORECASE)) compile_count += 1 weight_patterns.append(tuple(result)) self.regex[group_name].append((weight, weight_patterns)) Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
def run(self, titles): titles = distinct([simplify(title) for title in titles]) Logr.info(str(titles)) Logr.debug("------------------------------------------------------------") root, tails = self.parse(titles) Logr.debug("--------------------------PARSE-----------------------------") for node in root: print_tree(node) Logr.debug("--------------------------MERGE-----------------------------") self.merge(root) Logr.debug("--------------------------FINAL-----------------------------") for node in root: print_tree(node) Logr.debug("--------------------------RESULT-----------------------------") scores = {} results = [] for tail in tails: score, value, original_value = tail.full_value() if value in scores: scores[value] += score else: results.append((value, original_value)) scores[value] = score Logr.debug("%s %s %s", score, value, original_value) sorted_results = sorted(results, key=lambda item: (scores[item[0]], item[1]), reverse = True) return [result[0] for result in sorted_results]
def construct_patterns(self, pattern_groups): compile_start = datetime.now() compile_count = 0 for group_name, patterns in pattern_groups: if group_name not in self.regex: self.regex[group_name] = [] # Transform into weight groups if type(patterns[0]) is str or type(patterns[0][0]) not in [int, float]: patterns = [(1.0, patterns)] for weight, patterns in patterns: weight_patterns = [] for pattern in patterns: # Transform into multi-fragment patterns if type(pattern) is str: pattern = (pattern,) if type(pattern) is tuple and len(pattern) == 2: if type(pattern[0]) is str and is_list_type(pattern[1], str): pattern = (pattern,) result = [] for value in pattern: if type(value) is tuple: if len(value) == 2: # Construct OR-list pattern value = value[0] % '|'.join(value[1]) elif len(value) == 1: value = value[0] result.append(re.compile(value, re.IGNORECASE)) compile_count += 1 weight_patterns.append(tuple(result)) self.regex[group_name].append((weight, weight_patterns)) Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
def construct_patterns(self, pattern_groups): compile_start = datetime.now() compile_count = 0 for group_name, patterns in pattern_groups: if group_name not in self.regex: self.regex[group_name] = [] # Transform into weight groups if type(patterns[0]) is str or type(patterns[0][0]) not in [int, float]: patterns = [(1.0, patterns)] for weight, patterns in patterns: weight_patterns = [] for pattern in [CaperPattern.construct(v) for v in patterns if v]: compile_count += pattern.compile() weight_patterns.append(pattern) self.regex[group_name].append((weight, weight_patterns)) Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
def load(self): self.load_directory('sources', Source) Logr.info("Finished loading modules")