def parse_closure(self, parent_head, subject): parent_node = parent_head[0] if type( parent_head) is list else parent_head nodes, match = self.match(parent_head, parent_node, subject) # Capturing broke on constraint, return now if not match: return nodes Logr.debug('created closure node with subject.value: "%s"' % subject.value) result = [CaperClosureNode(subject, parent_head, match)] # Branch if the match was indefinite (weight below 1.0) if match.result and match.weight < 1.0: if match.num_fragments == 1: result.append(CaperClosureNode(subject, parent_head)) else: nodes.append(CaperClosureNode(subject, parent_head)) nodes.append(result[0] if len(result) == 1 else result) return nodes
def parse_closure(self, parent_head, subject): parent_node = parent_head[0] if type(parent_head) is list else parent_head nodes, match = self.match(parent_head, parent_node, subject) # Capturing broke on constraint, return now if not match: return nodes Logr.debug('created closure node with subject.value: "%s"' % subject.value) result = [CaperClosureNode( subject, parent_head, match )] # Branch if the match was indefinite (weight below 1.0) if match.result and match.weight < 1.0: if match.num_fragments == 1: result.append(CaperClosureNode(subject, parent_head)) else: nodes.append(CaperClosureNode(subject, parent_head)) nodes.append(result[0] if len(result) == 1 else result) return nodes
def calculate_sim_links(for_node, other_nodes): for node in other_nodes: if node in for_node.links: continue Logr.debug('calculating similarity between "%s" and "%s"', for_node.value, node.value) # Get similarity similarity_matcher = create_matcher(for_node.value, node.value) similarity = similarity_matcher.quick_ratio() # Get for_node -> node opcodes a_opcodes_matcher = create_matcher(for_node.value, node.value, swap_longest = False) a_opcodes = a_opcodes_matcher.get_opcodes() a_stats = get_opcode_stats(for_node, node, a_opcodes) Logr.debug('-' * 100) # Get node -> for_node opcodes b_opcodes_matcher = create_matcher(node.value, for_node.value, swap_longest = False) b_opcodes = b_opcodes_matcher.get_opcodes() b_stats = get_opcode_stats(for_node, node, b_opcodes) for_node.links[node] = SimLink(similarity, a_opcodes, a_stats) node.links[for_node] = SimLink(similarity, b_opcodes, b_stats)
def __init__(self, debug=False): if not SceneParser.matcher: SceneParser.matcher = Matcher(PATTERN_GROUPS) Logr.info("Fragment matcher for %s created", self.__class__.__name__) super(SceneParser, self).__init__(SceneParser.matcher, debug)
def get_usage(self, month=None, year=None): if not month: month = datetime.date.today().month if not year: year = datetime.date.today().year # Check if we have already cached the result obj_hash = self.cache_hash(month, year) if self.cache_valid(obj_hash): return self.cache[obj_hash]['result'] # Ensure we are logged in if not self.conn: self.login() result = None try: Logr.info("Retrieving latest data usage details") usage = self.conn.get_usage(month, year) result = { 'total': usage.total, 'days': usage.days } except CompassConnectionException: pass # Cache the result self.cache[obj_hash] = { 'result': result, 'date': datetime.datetime.utcnow() } return result
def parse(self, titles): root = [] tails = [] for title in titles: Logr.debug(title) cur = None words = title.split(' ') for wx in xrange(len(words)): word = strip(words[wx]) if cur is None: cur = find_node(root, word) if cur is None: cur = DNode(word, None, num_children=len(words) - wx, original_value=title) root.append(cur) else: parent = cur parent.weight += 1 cur = find_node(parent.right, word) if cur is None: Logr.debug("%s %d", word, len(words) - wx) cur = DNode(word, parent, num_children=len(words) - wx) sorted_append(parent.right, cur, lambda a: a.num_children < cur.num_children) else: cur.weight += 1 tails.append(cur) return root, tails
def test_merge_is_order_independent(self): root_one = [ self._create_chain(['avatar', 'the', 'legend', 'of', 'korra']), self._create_chain(['la', 'leggenda', 'di', 'korra']), self._create_chain(['the', 'last', 'airbender', 'the', 'legend', 'of', 'korra']) ] self._create_chain(['legend', 'of', 'korra'], root_one[-1]) root_one.append(self._create_chain(['legend', 'of', 'korra'])) result_one = self.merge.merge(root_one) Logr.debug("-----------------------------------------------------------------") root_two = [ self._create_chain(['the', 'legend', 'of', 'korra']), ] self._create_chain(['last', 'airbender', 'the', 'legend', 'of', 'korra'], root_two[-1]) root_two += [ self._create_chain(['legend', 'of', 'korra']), self._create_chain(['la', 'leggenda', 'di', 'korra']), self._create_chain(['avatar', 'the', 'legend', 'of', 'korra']) ] result_two = self.merge.merge(root_two) Logr.debug("=================================================================") assert itemsMatch( self._get_chain_values(result_one), self._get_chain_values(result_two) )
def _merge(self, nodes, depth = 0): Logr.debug(str('\t' * depth) + str(nodes)) if not len(nodes): return [] top = nodes[0] # Merge into top for x in range(len(nodes)): # Merge extra results into top if x > 0: top.value = None top.weight += nodes[x].weight self.destroy_nodes_right(top.right) if len(nodes[x].right): top.join_right(nodes[x].right) Logr.debug("= %s joined %s", nodes[x], top) nodes[x].dead = True nodes = [n for n in nodes if not n.dead] # Traverse further for node in nodes: if len(node.right): node.right = self._merge(node.right, depth + 1) return nodes
def merge(self, root): for x in range(len(root)): Logr.debug(root[x]) root[x].right = self._merge(root[x].right) Logr.debug('=================================================================') return root
def capture_closure(self, tag, regex=None, func=None, single=True, **kwargs): Logr.debug('capture_closure("%s", "%s", %s, %s)', tag, regex, func, single) if self.step_source != 'closure': if self.step_source is None: self.step_source = 'closure' else: raise ValueError( "Unable to mix fragment and closure capturing in a group") self.steps.append( CaptureStep(self, tag, 'closure', regex=regex, func=func, single=single, **kwargs)) return self
def print_tree(node, depth = 0): Logr.debug(str('\t' * depth) + str(node)) if len(node.right): for child in node.right: print_tree(child, depth + 1) else: Logr.debug(node.full_value()[1])
def load(self): parser = ConfigParser.ConfigParser() parser.read(os.path.join(self.base_dir, 'data.cfg')) for module_name, spec in self.module_loader.modules.items(): if parser.has_section(module_name): self.load_module(parser, module_name, spec) else: Logr.debug("no section named '%s'" % module_name)
def process(self, matches): matches = [match.groupdict() for match in matches] if self.transform: try: return self.transform(matches) except Exception as ex: Logr.warn('Exception raised while transforming matches: %s', ex) return matches
def parse_subject(self, parent_head, subject): Logr.debug("parse_subject (%s) subject: %s", self.step_source, repr(subject)) if type(subject) is CaperClosure: return self.parse_closure(parent_head, subject) if type(subject) is CaperFragment: return self.parse_fragment(parent_head, subject) raise ValueError('Unknown subject (%s)', subject)
def fragment_match(self, fragment, group_name=None): """Follow a fragment chain to try find a match :type fragment: caper.objects.CaperFragment :type group_name: str or None :return: The weight of the match found between 0.0 and 1.0, where 1.0 means perfect match and 0.0 means no match :rtype: (float, dict, int) """ group_name, weight_groups = self.find_group(group_name) for weight, patterns in weight_groups: for pattern in patterns: success = True result = {} num_matched = 0 fragment_iterator = fragment.take_right( return_type='value', include_separators=pattern.include_separators, include_source=True ) for subject, fragment_pattern in itertools.izip_longest(fragment_iterator, pattern): # No patterns left to match if not fragment_pattern: break # No fragments left to match against pattern if not subject: success = False break value, source = subject matches = pattern.execute(fragment_pattern, value) if matches: for match in pattern.process(matches): update_dict(result, match) else: success = False break if source == 'subject': num_matched += 1 if success: Logr.debug('Found match with weight %s using regex pattern "%s"' % (weight, [sre.pattern for sre in pattern.patterns])) return float(weight), result, num_matched return 0.0, None, 1
def print_link_tree(nodes): for node in nodes: Logr.debug(node.value) Logr.debug('\tnum_merges: %s', node.num_merges) if len(node.links): Logr.debug('\t========== LINKS ==========') for link_node, link in node.links.items(): Logr.debug('\t%0.2f -- %s', link.similarity, link_node.value) Logr.debug('\t---------------------------')
def kill_nodes_above(nodes, above_sim): killed_nodes = [] for node in nodes: if node.dead: continue Logr.debug(node.value) for link_node, link in node.links.items(): if link_node.dead: continue Logr.debug('\t%0.2f -- %s', link.similarity, link_node.value) if link.similarity >= above_sim: if len(link_node.value) > len(node.value): Logr.debug('\t\tvery similar, killed this node') link_node.dead = True node.num_merges += 1 killed_nodes.append(link_node) else: Logr.debug('\t\tvery similar, killed owner') node.dead = True link_node.num_merges += 1 killed_nodes.append(node) kill_nodes(nodes, killed_nodes)
def parse(self, name, parser='scene'): closures = self._closure_split(name) closures = self._fragment_split(closures) # Print closures for closure in closures: Logr.debug("closure [%s]", closure.value) for fragment in closure.fragments: Logr.debug("\tfragment [%s]", fragment.value) if parser not in self.parsers: raise ValueError("Unknown parser") # TODO autodetect the parser type return self.parsers[parser](self.debug).run(closures)
def check_constraints(self, constraints, parent_head, subject, **kwargs): parent_node = parent_head[0] if type(parent_head) is list else parent_head # Check constraints for constraint in [c for c in constraints if c.target == subject.__key__ or not c.target]: Logr.debug("Testing constraint %s against subject %s", repr(constraint), repr(subject)) weight, success = constraint.execute(parent_node, subject, **kwargs) if success: Logr.debug('capturing broke on "%s" at %s', subject.value, constraint) parent_node.finished_groups.append(self) return True, weight == 1.0 return False, None
def kill_trailing_nodes(nodes): killed_nodes = [] for node in nodes: if node.dead: continue Logr.debug(node.value) for link_node, link in node.links.items(): if link_node.dead: continue is_valid = link.stats.get('valid', False) has_deletions = False has_insertions = False has_replacements = False for opcode in link.opcodes: if opcode[0] == 'delete': has_deletions = True if opcode[0] == 'insert': has_insertions = True if opcode[0] == 'replace': has_replacements = True equal_perc = link.stats.get('equal', 0) / float(len(node.value)) insert_perc = link.stats.get('insert', 0) / float(len(node.value)) Logr.debug('\t({0:<24}) [{1:02d}:{2:02d} = {3:02d} {4:3.0f}% {5:3.0f}%] -- {6:<45}'.format( 'd:%s, i:%s, r:%s' % (has_deletions, has_insertions, has_replacements), len(node.value), len(link_node.value), link.stats.get('equal', 0), equal_perc * 100, insert_perc * 100, '"{0}"'.format(link_node.value) )) Logr.debug('\t\t%s', link.stats) kill = all([ is_valid, equal_perc >= 0.5, insert_perc < 2, has_insertions, not has_deletions, not has_replacements ]) if kill: Logr.debug('\t\tkilled this node') link_node.dead = True node.num_merges += 1 killed_nodes.append(link_node) kill_nodes(nodes, killed_nodes)
def post(self, widget_id, client_id=None, **kwargs): # Create message message = kwargs message.update({ 'id': widget_id, 'updatedAt': int(time.time() + 300) }) # Queue for clients to send clients = self.clients.items() if client_id: clients = [(client_id, self.clients[client_id])] for cid, client in clients: client['messages'][client['next_mid']] = message client['next_mid'] += 1 Logr.debug("Message posted, queued to be sent to %s clients" % len(self.clients))
def construct_patterns(self, pattern_groups): compile_start = datetime.now() compile_count = 0 for group_name, patterns in pattern_groups: if group_name not in self.regex: self.regex[group_name] = [] # Transform into weight groups if type(patterns[0]) is str or type( patterns[0][0]) not in [int, float]: patterns = [(1.0, patterns)] for weight, patterns in patterns: weight_patterns = [] for pattern in patterns: # Transform into multi-fragment patterns if type(pattern) is str: pattern = (pattern, ) if type(pattern) is tuple and len(pattern) == 2: if type(pattern[0]) is str and is_list_type( pattern[1], str): pattern = (pattern, ) result = [] for value in pattern: if type(value) is tuple: if len(value) == 2: # Construct OR-list pattern value = value[0] % '|'.join(value[1]) elif len(value) == 1: value = value[0] result.append(re.compile(value, re.IGNORECASE)) compile_count += 1 weight_patterns.append(tuple(result)) self.regex[group_name].append((weight, weight_patterns)) Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
def capture_closure(self, tag, regex=None, func=None, single=True, **kwargs): Logr.debug('capture_closure("%s", "%s", %s, %s)', tag, regex, func, single) if self.step_source != 'closure': if self.step_source is None: self.step_source = 'closure' else: raise ValueError("Unable to mix fragment and closure capturing in a group") self.steps.append(CaptureStep( self, tag, 'closure', regex=regex, func=func, single=single, **kwargs )) return self
def fragment_match(self, fragment, group_name=None): """Follow a fragment chain to try find a match :type fragment: caper.objects.CaperFragment :type group_name: str or None :return: The weight of the match found between 0.0 and 1.0, where 1.0 means perfect match and 0.0 means no match :rtype: (float, dict, int) """ group_name, weight_groups = self.find_group(group_name) for weight, patterns in weight_groups: for pattern in patterns: cur_fragment = fragment success = True result = {} # Ignore empty patterns if len(pattern) < 1: break for fragment_pattern in pattern: if not cur_fragment: success = False break match = fragment_pattern.match(cur_fragment.value) if match: update_dict(result, match.groupdict()) else: success = False break cur_fragment = cur_fragment.right if cur_fragment else None if success: Logr.debug("Found match with weight %s" % weight) return float(weight), result, len(pattern) return 0.0, None, 1
def compile(self): patterns = self.patterns self.patterns = [] for pattern in patterns: if type(pattern) is tuple: if len(pattern) == 2: # Construct OR-list pattern pattern = pattern[0] % '|'.join(pattern[1]) elif len(pattern) == 1: pattern = pattern[0] # Compile the pattern try: self.patterns.append(re.compile(pattern, re.IGNORECASE)) except Exception as ex: Logr.error('Unable to compile pattern "%s"', pattern) raise ex return len(patterns)
def construct_patterns(self, pattern_groups): compile_start = datetime.now() compile_count = 0 for group_name, patterns in pattern_groups: if group_name not in self.regex: self.regex[group_name] = [] # Transform into weight groups if type(patterns[0]) is str or type(patterns[0][0]) not in [int, float]: patterns = [(1.0, patterns)] for weight, patterns in patterns: weight_patterns = [] for pattern in patterns: # Transform into multi-fragment patterns if type(pattern) is str: pattern = (pattern,) if type(pattern) is tuple and len(pattern) == 2: if type(pattern[0]) is str and is_list_type(pattern[1], str): pattern = (pattern,) result = [] for value in pattern: if type(value) is tuple: if len(value) == 2: # Construct OR-list pattern value = value[0] % '|'.join(value[1]) elif len(value) == 1: value = value[0] result.append(re.compile(value, re.IGNORECASE)) compile_count += 1 weight_patterns.append(tuple(result)) self.regex[group_name].append((weight, weight_patterns)) Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
def load_directory(self, sub_dir, base_type): # Load directory module first m = __import__(sub_dir) for module_name in getattr(self, sub_dir): Logr.info("Loading '%s.%s'" % (sub_dir, module_name)) __import__(sub_dir + '.' + module_name) module = getattr(m, module_name) # Find classes that extend the 'base_type' base class module_class = None for name, obj in inspect.getmembers(module, inspect.isclass): if obj is not base_type and issubclass(obj, base_type): if module_class is None: module_class = obj else: Logr.warning("Only one class is allowed per module") if module_class is not None: spec = { 'module_name': sub_dir + '.' + module_name, 'module': module, 'class': module_class, 'detail': module_class.__detail__, 'options': module_class.__options__ } getattr(self, sub_dir)[module_name] = spec self.modules[sub_dir + '.' + module_name] = spec else: Logr.warning("Unable to find class inside module '%s.%s'" % (sub_dir, module_name))
def construct_patterns(self, pattern_groups): compile_start = datetime.now() compile_count = 0 for group_name, patterns in pattern_groups: if group_name not in self.regex: self.regex[group_name] = [] # Transform into weight groups if type(patterns[0]) is str or type(patterns[0][0]) not in [int, float]: patterns = [(1.0, patterns)] for weight, patterns in patterns: weight_patterns = [] for pattern in [CaperPattern.construct(v) for v in patterns if v]: compile_count += pattern.compile() weight_patterns.append(pattern) self.regex[group_name].append((weight, weight_patterns)) Logr.info("Compiled %s patterns in %ss", compile_count, delta_seconds(datetime.now() - compile_start))
def parse(self, name, parser='scene'): if not name: return None closures = self._closure_split(name) closures = self._fragment_split(closures) # Print closures for closure in closures: Logr.debug("closure [%s]", closure.value) for fragment in closure.fragments: Logr.debug("\tfragment (%s)[%s](%s)", fragment.left_sep, fragment.value, fragment.right_sep) if parser not in self.parsers: raise ValueError("Unknown parser") if not closures: return None # TODO autodetect the parser type return self.parsers[parser](self.debug).run(closures)
def check_constraints(self, constraints, parent_head, subject, **kwargs): parent_node = parent_head[0] if type( parent_head) is list else parent_head # Check constraints for constraint in [ c for c in constraints if c.target == subject.__key__ or not c.target ]: Logr.debug("Testing constraint %s against subject %s", repr(constraint), repr(subject)) weight, success = constraint.execute(parent_node, subject, **kwargs) if success: Logr.debug('capturing broke on "%s" at %s', subject.value, constraint) parent_node.finished_groups.append(self) return True, weight == 1.0 return False, None
def read(self): client_id = self.next_id self.next_id += 1 self.clients[client_id] = { 'next_mid': 1, 'messages': {} } for func in self.functions: func(client_id=client_id) while client_id in self.clients: sent = [] for mid, message in self.clients[client_id]['messages'].items(): Logr.debug("Sent message: %s" % message) yield "data: %s\n\n" % json.dumps(message) sent.append(mid) # Remove sent messages for mid in sent: self.clients[client_id]['messages'].pop(mid) time.sleep(5)
def match(self, parent_head, parent_node, subject): nodes = [] # Check pre constaints broke, definite = self.check_constraints(self.pre_constraints, parent_head, subject) if broke: nodes.append(parent_head) if definite: return nodes, None # Try match subject against the steps available match = None for step in self.steps: if step.source == 'closure' and type(subject) is not CaperClosure: pass elif step.source == 'fragment' and type(subject) is CaperClosure: Logr.debug( 'Closure encountered on fragment step, jumping into fragments' ) return [CaperClosureNode(subject, parent_head, None)], None match = step.execute(subject) if match.success: if type(match.result) is dict: match.result = clean_dict(match.result) Logr.debug( 'Found match with weight %s, match: %s, num_fragments: %s' % (match.weight, match.result, match.num_fragments)) step.matched = True break if all([step.single and step.matched for step in self.steps]): Logr.debug('All steps completed, group finished') parent_node.finished_groups.append(self) return nodes, match # Check post constraints broke, definite = self.check_constraints(self.post_constraints, parent_head, subject, match=match) if broke: return nodes, None return nodes, match
def build(self): max_matched = 0 for head in self.heads: for chain in self.combine_chain(head): if chain.num_matched > max_matched: max_matched = chain.num_matched self.chains.append(chain) for chain in self.chains: chain.weights.append(chain.num_matched / float(max_matched or chain.num_matched or 1)) chain.finish() self.chains.sort(key=lambda chain: chain.weight, reverse=True) for chain in self.chains: Logr.debug("chain weight: %.02f", chain.weight) Logr.debug("\tInfo: %s", chain.info) Logr.debug("\tWeights: %s", chain.weights) Logr.debug("\tNumber of Fragments Matched: %s", chain.num_matched)
def execute(self, fragment): """Execute step on fragment :type fragment: CaperFragment :rtype : CaptureMatch """ match = CaptureMatch(self.tag, self) if self.regex: weight, result, num_fragments = self.capture_group.parser.matcher.fragment_match( fragment, self.regex) Logr.debug('(execute) [regex] tag: "%s"', self.tag) if not result: return match # Populate CaptureMatch match.success = True match.weight = weight match.result = result match.num_fragments = num_fragments elif self.func: result = self.func(fragment) Logr.debug('(execute) [func] %s += "%s"', self.tag, match) if not result: return match # Populate CaptureMatch match.success = True match.weight = 1.0 match.result = result else: Logr.debug('(execute) [raw] %s += "%s"', self.tag, fragment.value) include_separators = self.kwargs.get('include_separators', False) # Populate CaptureMatch match.success = True match.weight = 1.0 if include_separators: match.result = (fragment.left_sep, fragment.value, fragment.right_sep) else: match.result = fragment.value return match
def match(self, parent_head, parent_node, subject): nodes = [] # Check pre constaints broke, definite = self.check_constraints(self.pre_constraints, parent_head, subject) if broke: nodes.append(parent_head) if definite: return nodes, None # Try match subject against the steps available match = None for step in self.steps: if step.source == 'closure' and type(subject) is not CaperClosure: pass elif step.source == 'fragment' and type(subject) is CaperClosure: Logr.debug('Closure encountered on fragment step, jumping into fragments') return [CaperClosureNode(subject, parent_head, None)], None match = step.execute(subject) if match.success: if type(match.result) is dict: match.result = clean_dict(match.result) Logr.debug('Found match with weight %s, match: %s, num_fragments: %s' % ( match.weight, match.result, match.num_fragments )) step.matched = True break if all([step.single and step.matched for step in self.steps]): Logr.debug('All steps completed, group finished') parent_node.finished_groups.append(self) return nodes, match # Check post constraints broke, definite = self.check_constraints(self.post_constraints, parent_head, subject, match=match) if broke: return nodes, None return nodes, match
def execute(self, fragment): """Execute step on fragment :type fragment: CaperFragment :rtype : CaptureMatch """ match = CaptureMatch(self.tag, self) if self.regex: weight, result, num_fragments = self.capture_group.parser.matcher.fragment_match(fragment, self.regex) Logr.debug('(execute) [regex] tag: "%s"', self.tag) if not result: return match # Populate CaptureMatch match.success = True match.weight = weight match.result = result match.num_fragments = num_fragments elif self.func: result = self.func(fragment) Logr.debug('(execute) [func] %s += "%s"', self.tag, match) if not result: return match # Populate CaptureMatch match.success = True match.weight = 1.0 match.result = result else: Logr.debug('(execute) [raw] %s += "%s"', self.tag, fragment.value) include_separators = self.kwargs.get('include_separators', False) # Populate CaptureMatch match.success = True match.weight = 1.0 if include_separators: match.result = (fragment.left_sep, fragment.value, fragment.right_sep) else: match.result = fragment.value return match
def get_opcode_stats(for_node, node, opcodes): stats = {} for tag, i1, i2, j1, j2 in opcodes: Logr.debug(stats_print_format.format( tag, i1, i2, j1, j2 )) if tag in ['insert', 'delete']: ax = None, None bx = None, None if tag == 'insert': ax = get_indices(for_node.value, i1 - 1, i1) bx = get_indices(node.value, j1, j2 - 1) if tag == 'delete': ax = get_indices(for_node.value, j1 - 1, j1) bx = get_indices(node.value, i1, i2 - 1) av = get_index_values(for_node.value, *ax) bv = get_index_values(node.value, *bx) Logr.debug( '\t\t%s %s [%s><%s] <---> %s %s [%s><%s]', ax, av, av[0], av[1], bx, bv, bv[0], bv[1] ) head_valid = av[0] in [None, ' '] or bv[0] in [None, ' '] tail_valid = av[1] in [None, ' '] or bv[1] in [None, ' '] valid = head_valid and tail_valid if 'valid' not in stats or (stats['valid'] and not valid): stats['valid'] = valid Logr.debug('\t\t' + ('VALID' if valid else 'INVALID')) if tag not in stats: stats[tag] = 0 stats[tag] += (i2 - i1) or (j2 - j1) return stats
def print_tree(self, heads): if not self.debug: return for head in heads: head = head if type(head) is list else [head] if type(head[0]) is CaperFragmentNode: for fragment in head[0].fragments: Logr.debug(fragment.value) else: Logr.debug(head[0].closure.value) for node in head: Logr.debug('\t' + str(node).ljust(55) + '\t' + ( str(node.match.weight) + '\t' + str(node.match.result) ) if node.match else '') if len(head) > 0 and head[0].parent: self.print_tree([head[0].parent])
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from helpers import setup_path setup_path() import logging from logr import Logr Logr.configure(logging.DEBUG) from caper import Caper from matchers import has_info from hamcrest import * caper = Caper() def test_resolution(): assert_that(caper.parse('Show Name.S01E05.720p-GROUP'), has_info('video', {'resolution': '720p'})) assert_that(caper.parse('Show Name.S01E05.1080p'), has_info('video', {'resolution': '1080p'}))
def execute(self): heads_finished = None while heads_finished is None or not (len(heads_finished) == len( self.result.heads) and all(heads_finished)): heads_finished = [] heads = self.result.heads self.result.heads = [] for head in heads: node = head[0] if type(head) is list else head if self in node.finished_groups: Logr.debug("head finished for group") self.result.heads.append(head) heads_finished.append(True) continue Logr.debug('') Logr.debug(node) next_subject = node.next() Logr.debug( '----------[%s] (%s)----------' % (next_subject, repr(next_subject.value) if next_subject else None)) if next_subject: for node_result in self.parse_subject(head, next_subject): self.result.heads.append(node_result) Logr.debug('Heads: %s', self.result.heads) heads_finished.append(self in node.finished_groups or next_subject is None) if len(self.result.heads) == 0: self.result.heads = heads Logr.debug("heads_finished: %s, self.result.heads: %s", heads_finished, self.result.heads) Logr.debug("group finished")
tests = CaperTests(debug=arguments['logging_mode'] == 'debug') parser_type = arguments['parser_type'] or \ raw_input_default('Parser type (scene, anime) [scene]: ', 'scene') test_file = '' test_file_default = 'scene.csv' if os.path.isfile(parser_type + '.csv'): test_file_default = parser_type + '.csv' while test_file == '': test_file = arguments['test_file'] or \ raw_input_default('Test file [%s]: ' % test_file_default, test_file_default) if not os.path.isfile(test_file): test_file = '' print "ERROR: Test file does not exist" print logging_mode = arguments['logging_mode'] or \ raw_input_default('Logging mode (debug, info) [info]: ', 'info') if logging_mode == 'debug': Logr.configure(logging.DEBUG, trace_origin=True) else: Logr.configure(logging.INFO) tests.load(test_file, 100) tests.run(parser_type, arguments)