def __init__(self, debug=False):
if not SceneParser.matcher:
SceneParser.matcher = Matcher(PATTERN_GROUPS)
Logr.info("Fragment matcher for %s created",
self.__class__.__name__)
super(SceneParser, self).__init__(SceneParser.matcher, debug)
def get_usage(self, month=None, year=None):
if not month:
month = datetime.date.today().month
if not year:
year = datetime.date.today().year
# Check if we have already cached the result
obj_hash = self.cache_hash(month, year)
if self.cache_valid(obj_hash):
return self.cache[obj_hash]['result']
# Ensure we are logged in
if not self.conn:
self.login()
result = None
try:
Logr.info("Retrieving latest data usage details")
usage = self.conn.get_usage(month, year)
result = {
'total': usage.total,
'days': usage.days
}
except CompassConnectionException:
pass
# Cache the result
self.cache[obj_hash] = {
'result': result,
'date': datetime.datetime.utcnow()
}
return result
def load_directory(self, sub_dir, base_type):
# Load directory module first
m = __import__(sub_dir)
for module_name in getattr(self, sub_dir):
Logr.info("Loading '%s.%s'" % (sub_dir, module_name))
__import__(sub_dir + '.' + module_name)
module = getattr(m, module_name)
# Find classes that extend the 'base_type' base class
module_class = None
for name, obj in inspect.getmembers(module, inspect.isclass):
if obj is not base_type and issubclass(obj, base_type):
if module_class is None:
module_class = obj
else:
Logr.warning("Only one class is allowed per module")
if module_class is not None:
spec = {
'module_name': sub_dir + '.' + module_name,
'module': module,
'class': module_class,
'detail': module_class.__detail__,
'options': module_class.__options__
}
getattr(self, sub_dir)[module_name] = spec
self.modules[sub_dir + '.' + module_name] = spec
else:
Logr.warning("Unable to find class inside module '%s.%s'" % (sub_dir, module_name))
def construct_patterns(self, pattern_groups):
compile_start = datetime.now()
compile_count = 0
for group_name, patterns in pattern_groups:
if group_name not in self.regex:
self.regex[group_name] = []
# Transform into weight groups
if type(patterns[0]) is str or type(
patterns[0][0]) not in [int, float]:
patterns = [(1.0, patterns)]
for weight, patterns in patterns:
weight_patterns = []
for pattern in patterns:
# Transform into multi-fragment patterns
if type(pattern) is str:
pattern = (pattern, )
if type(pattern) is tuple and len(pattern) == 2:
if type(pattern[0]) is str and is_list_type(
pattern[1], str):
pattern = (pattern, )
result = []
for value in pattern:
if type(value) is tuple:
if len(value) == 2:
# Construct OR-list pattern
value = value[0] % '|'.join(value[1])
elif len(value) == 1:
value = value[0]
result.append(re.compile(value, re.IGNORECASE))
compile_count += 1
weight_patterns.append(tuple(result))
self.regex[group_name].append((weight, weight_patterns))
Logr.info("Compiled %s patterns in %ss", compile_count,
delta_seconds(datetime.now() - compile_start))
def run(self, titles):
titles = distinct([simplify(title) for title in titles])
Logr.info(str(titles))
Logr.debug("------------------------------------------------------------")
root, tails = self.parse(titles)
Logr.debug("--------------------------PARSE-----------------------------")
for node in root:
print_tree(node)
Logr.debug("--------------------------MERGE-----------------------------")
self.merge(root)
Logr.debug("--------------------------FINAL-----------------------------")
for node in root:
print_tree(node)
Logr.debug("--------------------------RESULT-----------------------------")
scores = {}
results = []
for tail in tails:
score, value, original_value = tail.full_value()
if value in scores:
scores[value] += score
else:
results.append((value, original_value))
scores[value] = score
Logr.debug("%s %s %s", score, value, original_value)
sorted_results = sorted(results, key=lambda item: (scores[item[0]], item[1]), reverse = True)
return [result[0] for result in sorted_results]
def construct_patterns(self, pattern_groups):
compile_start = datetime.now()
compile_count = 0
for group_name, patterns in pattern_groups:
if group_name not in self.regex:
self.regex[group_name] = []
# Transform into weight groups
if type(patterns[0]) is str or type(patterns[0][0]) not in [int, float]:
patterns = [(1.0, patterns)]
for weight, patterns in patterns:
weight_patterns = []
for pattern in patterns:
# Transform into multi-fragment patterns
if type(pattern) is str:
pattern = (pattern,)
if type(pattern) is tuple and len(pattern) == 2:
if type(pattern[0]) is str and is_list_type(pattern[1], str):
pattern = (pattern,)
result = []
for value in pattern:
if type(value) is tuple:
if len(value) == 2:
# Construct OR-list pattern
value = value[0] % '|'.join(value[1])
elif len(value) == 1:
value = value[0]
result.append(re.compile(value, re.IGNORECASE))
compile_count += 1
weight_patterns.append(tuple(result))
self.regex[group_name].append((weight, weight_patterns))
Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
def construct_patterns(self, pattern_groups):
compile_start = datetime.now()
compile_count = 0
for group_name, patterns in pattern_groups:
if group_name not in self.regex:
self.regex[group_name] = []
# Transform into weight groups
if type(patterns[0]) is str or type(patterns[0][0]) not in [int, float]:
patterns = [(1.0, patterns)]
for weight, patterns in patterns:
weight_patterns = []
for pattern in [CaperPattern.construct(v) for v in patterns if v]:
compile_count += pattern.compile()
weight_patterns.append(pattern)
self.regex[group_name].append((weight, weight_patterns))
Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
def __init__(self, debug=False):
if not SceneParser.matcher:
SceneParser.matcher = Matcher(PATTERN_GROUPS)
Logr.info("Fragment matcher for %s created", self.__class__.__name__)
super(SceneParser, self).__init__(SceneParser.matcher, debug)
def load(self):
self.load_directory('sources', Source)
Logr.info("Finished loading modules")
