Ejemplo n.º 1
0
    def test_tokenize_equals(self):
        s = 'a b=c'
        t = list(options.tokenize(s))
        expected = [ts(0, 1, 'a'), ts(2, 3, 'b'), ts(3, 5, '=c')]
        self.assertTokensEquals(s, t, expected, ('a', 'b', '=c'))

        s = 'a b =c'
        t = list(options.tokenize(s))
        expected = [ts(0, 1, 'a'), ts(2, 3, 'b'), ts(4, 6, '=c')]
        self.assertTokensEquals(s, t, expected, ('a', 'b', '=c'))

        s = 'a b= c'
        t = list(options.tokenize(s))
        expected = [ts(0, 1, 'a'), ts(2, 3, 'b'), ts(3, 4, '='), ts(5, 6, 'c')]
        self.assertTokensEquals(s, t, expected, ('a', 'b', '=', 'c'))

        s = 'a b = c'
        t = list(options.tokenize(s))
        expected = [ts(0, 1, 'a'), ts(2, 3, 'b'), ts(4, 5, '='), ts(6, 7, 'c')]
        self.assertTokensEquals(s, t, expected, ('a', 'b', '=', 'c'))

        s = 'a b  = c'
        t = list(options.tokenize(s))
        expected = [ts(0, 1, 'a'), ts(2, 3, 'b'), ts(5, 6, '='), ts(7, 8, 'c')]
        self.assertTokensEquals(s, t, expected, ('a', 'b', '=', 'c'))
Ejemplo n.º 2
0
    def test_parsing_error(self):
        s = 'no escaped character\\'
        message = r'No escaped character \(position 21, ...cter\\\)'
        with self.assertRaisesRegexp(errors.ParsingError, message):
            list(options.tokenize(s))

        s = 'no closing "quotation'
        message = r'No closing quotation \(position 21, ...ation\)'
        with self.assertRaisesRegexp(errors.ParsingError, message):
            list(options.tokenize(s))
Ejemplo n.º 3
0
    def test_tokenize(self):
        s = 'bar -x'
        t = list(options.tokenize(s))
        expected = [ts(0, 3, 'bar'), ts(4, 6, '-x')]
        self.assertTokensEquals(s, t, expected, ('bar', '-x'))

        s = 'wx    y =z '
        t = list(options.tokenize(s))
        expected = [ts(0, 2, 'wx'), ts(6, 7, 'y'), ts(8, 10, '=z')]
        self.assertTokensEquals(s, t, expected, ('wx', 'y', '=z'))

        s = "a 'b' c"
        t = list(options.tokenize(s))
        expected = [ts(0, 1, 'a'), ts(2, 5, 'b'), ts(6, 7, 'c')]
        self.assertTokensEquals(s, t, expected, ('a', "'b'", 'c'))

        s = "a 'b  ' c"
        t = list(options.tokenize(s))
        expected = [ts(0, 1, 'a'), ts(2, 7, 'b  '), ts(8, 9, 'c')]
        self.assertTokensEquals(s, t, expected, ('a', "'b  '", 'c'))
Ejemplo n.º 4
0
    def match(self):
        '''parse s and return a list of matchresult

        match works as follows:
        - tokenize the string using options.tokenize
        - look up a man page that matches the first token
        - check if the man page has 'subcommands', e.g. git commit, if so
          try to find a man page for the first two tokens
        - iterate all tokens
          - search the token as is
          - if the token is a short option (-abc) try to look up each option
            individually
          - if the previous match expected an arg, merge this token with
            the previous one
          - partialmatch: if this man page allows options to start without '-',
            try to match all characters individually (e.g. tar xzvf)
          - check if this man page has any positional arguments
          - mark this token as unknown

        after we finish processing all tokens, we:
        - merge unknown consecutive matches to become one matchresult
        - merge adjacent matchresults that have the same help text (e.g. if we
          had -vvv)
        '''
        logger.info('matching string %r', self.s)
        self.tokens = util.peekable(options.tokenize(self.s))
        #logger.info('tokenized %r to %s', self.s, [t[1] for t in self.tokens])
        self.ts = None

        def attempt_fuzzy(chars):
            m = []
            if chars[0] == '-':
                tokens = [chars[0:2]] + list(chars[2:])
                considerarg = True
            else:
                tokens = list(chars)
                considerarg = False

            oldp = self.pos
            for t in tokens:
                op = t if t[0] == '-' else '-' + t
                option = self.find_option(op)
                if option:
                    if considerarg and not m and option.expectsarg:
                        logger.info(
                            'option %r expected an arg, taking the rest too',
                            option)
                        return [
                            matchresult(self.pos, self.pos + len(chars),
                                        option.text, None)
                        ]

                    mr = matchresult(self.pos, self.pos + len(t), option.text,
                                     None)
                    m.append(mr)
                else:
                    m.append(self.unknown(t, self.pos + len(t)))
                self.pos += len(t)
            self.pos = oldp
            return m

        self.pos, endpos, token = self.nexttoken()
        mps = self.findmanpages(token)
        self.manpage = mps[0]
        if self.manpage.multicommand and self.tokens.hasnext():
            try:
                multi = '%s %s' % (token, self.tokens.peek()[2])
                logger.info(
                    '%r is a multicommand, trying to get another token and look up %r',
                    self.manpage, multi)
                mps = self.findmanpages(multi)
                self.manpage = mps[0]
                self.nexttoken()
                token = multi
            except errors.ProgramDoesNotExist:
                logger.info('no manpage %r for multicommand %r', multi,
                            self.manpage)

        option = None
        matches = []
        matches.append(matchresult(0, len(token), self.manpage.synopsis, None))

        while self.tokens.hasnext():
            self.pos, endpos, token = self.nexttoken()
            logger.info('trying to match token: %r', token)

            self._prevoption = self._currentoption
            option = self.find_option(token)
            if option:
                logger.info('found an exact match for %r: %r', token, option)
                mr = matchresult(self.pos, self.ts.endpos, option.text, None)
                matches.append(mr)
            else:
                if token != '-' and token.startswith(
                        '-') and not token.startswith('--'):
                    logger.debug('looks like a short option')
                    if len(token) > 2:
                        logger.info("trying to split it up")
                        matches.extend(attempt_fuzzy(token))
                        self.pos += len(token)
                    else:
                        matches.append(self.unknown(token))
                elif self._prevoption and self._prevoption.expectsarg:
                    logger.info(
                        "previous option possibly expected an arg, and we can't"
                        " find an option to match the current token, assuming it's an arg"
                    )
                    ea = self._prevoption.expectsarg
                    possibleargs = ea if isinstance(ea, list) else []
                    take = True
                    if possibleargs and token not in possibleargs:
                        take = False
                        logger.info(
                            'token %r not in list of possible args %r for %r',
                            token, possibleargs, self._prevoption)
                    if take:
                        pmr = matches[-1]
                        mr = matchresult(pmr.start, self.ts.endpos, pmr.text,
                                         None)
                        matches[-1] = mr
                    else:
                        matches.append(self.unknown(token))
                elif self.manpage.partialmatch:
                    logger.info('attemping to do a partial match')

                    m = attempt_fuzzy(token)
                    if any(mm.unknown for mm in m):
                        logger.info('one of %r was unknown', token)
                        matches.append(self.unknown(token))
                    else:
                        matches += m
                        self.pos += len(token)
                elif self.manpage.arguments:
                    d = self.manpage.arguments
                    k = list(d.keys())[0]
                    logger.info('got arguments, using %r', k)
                    text = d[k]
                    mr = matchresult(self.pos, self.ts.endpos, text, None)
                    matches.append(mr)
                else:
                    matches.append(self.unknown(token))

        def debugmatch():
            s = '\n'.join([
                '%d) %r = %r' % (i, self.s[m.start:m.end], m.text)
                for i, m in enumerate(matches)
            ])
            return s

        logger.debug('%r matches:\n%s', self.s, debugmatch())

        matches = self._mergeunknowns(matches)
        matches = self._mergeadjacent(matches)

        # add matchresult.match to existing matches
        for i, m in enumerate(matches):
            assert m.end <= len(self.s), '%d %d' % (m.end, len(self.s))
            matches[i] = matchresult(m.start, m.end, m.text,
                                     self.s[m.start:m.end])

        r = [(self.manpage.name, matches)]
        for mp in mps[1:]:
            r.append((mp, None))
        return r
Ejemplo n.º 5
0
    def match(self):
        '''parse s and return a list of matchresult

        match works as follows:
        - tokenize the string using options.tokenize
        - look up a man page that matches the first token
        - check if the man page has 'subcommands', e.g. git commit, if so
          try to find a man page for the first two tokens
        - iterate all tokens
          - search the token as is
          - if the token is a short option (-abc) try to look up each option
            individually
          - if the previous match expected an arg, merge this token with
            the previous one
          - partialmatch: if this man page allows options to start without '-',
            try to match all characters individually (e.g. tar xzvf)
          - check if this man page has any positional arguments
          - mark this token as unknown

        after we finish processing all tokens, we:
        - merge unknown consecutive matches to become one matchresult
        - merge adjacent matchresults that have the same help text (e.g. if we
          had -vvv)
        '''
        logger.info('matching string %r', self.s)
        self.tokens = util.peekable(options.tokenize(self.s))
        #logger.info('tokenized %r to %s', self.s, [t[1] for t in self.tokens])
        self.ts = None

        def attempt_fuzzy(chars):
            m = []
            if chars[0] == '-':
                tokens = [chars[0:2]] + list(chars[2:])
                considerarg = True
            else:
                tokens = list(chars)
                considerarg = False

            oldp = self.pos
            for t in tokens:
                op = t if t[0] == '-' else '-' + t
                option = self.find_option(op)
                if option:
                    if considerarg and not m and option.expectsarg:
                        logger.info('option %r expected an arg, taking the rest too', option)
                        return [matchresult(self.pos, self.pos+len(chars), option.text, None)]

                    mr = matchresult(self.pos, self.pos+len(t), option.text, None)
                    m.append(mr)
                else:
                    m.append(self.unknown(t, self.pos+len(t)))
                self.pos += len(t)
            self.pos = oldp
            return m

        self.pos, endpos, token = self.nexttoken()
        mps = self.findmanpages(token)
        self.manpage = mps[0]
        if self.manpage.multicommand and self.tokens.hasnext():
            try:
                multi = '%s %s' % (token, self.tokens.peek()[2])
                logger.info('%r is a multicommand, trying to get another token and look up %r', self.manpage, multi)
                mps = self.findmanpages(multi)
                self.manpage = mps[0]
                self.nexttoken()
                token = multi
            except errors.ProgramDoesNotExist:
                logger.info('no manpage %r for multicommand %r', multi, self.manpage)

        option = None
        matches = []
        matches.append(matchresult(0, len(token), self.manpage.synopsis, None))

        while self.tokens.hasnext():
            self.pos, endpos, token = self.nexttoken()
            logger.info('trying to match token: %r', token)

            self._prevoption = self._currentoption
            option = self.find_option(token)
            if option:
                logger.info('found an exact match for %r: %r', token, option)
                mr = matchresult(self.pos, self.ts.endpos, option.text, None)
                matches.append(mr)
            else:
                if token != '-' and token.startswith('-') and not token.startswith('--'):
                    logger.debug('looks like a short option')
                    if len(token) > 2:
                        logger.info("trying to split it up")
                        matches.extend(attempt_fuzzy(token))
                        self.pos += len(token)
                    else:
                        matches.append(self.unknown(token))
                elif self._prevoption and self._prevoption.expectsarg:
                    logger.info("previous option possibly expected an arg, and we can't"
                            " find an option to match the current token, assuming it's an arg")
                    ea = self._prevoption.expectsarg
                    possibleargs = ea if isinstance(ea, list) else []
                    take = True
                    if possibleargs and token not in possibleargs:
                        take = False
                        logger.info('token %r not in list of possible args %r for %r',
                                    token, possibleargs, self._prevoption)
                    if take:
                        pmr = matches[-1]
                        mr = matchresult(pmr.start, self.ts.endpos, pmr.text, None)
                        matches[-1] = mr
                    else:
                        matches.append(self.unknown(token))
                elif self.manpage.partialmatch:
                    logger.info('attemping to do a partial match')

                    m = attempt_fuzzy(token)
                    if any(mm.unknown for mm in m):
                        logger.info('one of %r was unknown', token)
                        matches.append(self.unknown(token))
                    else:
                        matches += m
                        self.pos += len(token)
                elif self.manpage.arguments:
                    d = self.manpage.arguments
                    k = list(d.keys())[0]
                    logger.info('got arguments, using %r', k)
                    text = d[k]
                    mr = matchresult(self.pos, self.ts.endpos, text, None)
                    matches.append(mr)
                else:
                    matches.append(self.unknown(token))

        def debugmatch():
            s = '\n'.join(['%d) %r = %r' % (i, self.s[m.start:m.end], m.text) for i, m in enumerate(matches)])
            return s

        logger.debug('%r matches:\n%s', self.s, debugmatch())

        matches = self._mergeunknowns(matches)
        matches = self._mergeadjacent(matches)

        # add matchresult.match to existing matches
        for i, m in enumerate(matches):
            assert m.end <= len(self.s), '%d %d' % (m.end, len(self.s))
            matches[i] = matchresult(m.start, m.end, m.text, self.s[m.start:m.end])

        r = [(self.manpage.name, matches)]
        for mp in mps[1:]:
            r.append((mp, None))
        return r