Esempio n. 1
0
def query(cmd, *args):
    if cmd == 'versions':

        # Returns the list of indexed versions in the following format:
        # topmenu submenu tag
        # Example: v3 v3.1 v3.1-rc10
        versions = OrderedDict()

        for line in scriptLines('list-tags', '-h'):
            taginfo = decode(line).split(' ')
            num = len(taginfo)
            topmenu, submenu = 'FIXME', 'FIXME'

            if (num == 1):
                tag, = taginfo
            elif (num == 2):
                submenu,tag = taginfo
            elif (num ==3):
                topmenu,submenu,tag = taginfo

            if db.vers.exists(tag):
                if topmenu not in versions:
                    versions[topmenu] = OrderedDict()
                if submenu not in versions[topmenu]:
                    versions[topmenu][submenu] = []
                versions[topmenu][submenu].append(tag)

        return versions

    elif cmd == 'latest':

        # Returns the tag considered as the latest one
        previous = None
        tag = ''
        index = 0

        # If we get the same tag twice, we are at the oldest one
        while not db.vers.exists(tag) and previous != tag:
            previous = tag
            tag = decode(script('get-latest', str(index))).rstrip('\n')
            index += 1

        return tag

    elif cmd == 'type':

        # Returns the type (blob or tree) associated to
        # the given path. Example:
        # > ./query.py type v3.1-rc10 /Makefile
        # blob
        # > ./query.py type v3.1-rc10 /arch
        # tree

        version = args[0]
        path = args[1]
        return decode(script('get-type', version, path)).strip()

    elif cmd == 'exist':

        # Returns True if the requested file exists, overwise returns False

        version = args[0]
        path = args[1]

        dirname, filename = os.path.split(path)

        entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
        for entry in entries:
            fname = entry.split(" ")[1]

            if fname == filename:
                return True

        return False

    elif cmd == 'dir':

        # Returns the contents (trees or blobs) of the specified directory
        # Example: ./query.py dir v3.1-rc10 /arch

        version = args[0]
        path = args[1]
        entries_str =  decode(script('get-dir', version, path))
        return entries_str.split("\n")[:-1]

    elif cmd == 'file':

        # Returns the contents of the specified file
        # Tokens are marked for further processing
        # Example: ./query.py file v3.1-rc10 /Makefile

        version = args[0]
        path = args[1]

        filename = os.path.basename(path)
        family = lib.getFileFamily(filename)

        if family != None:
            buffer = BytesIO()
            tokens = scriptLines('tokenize-file', version, path, family)
            even = True

            prefix = b''
            if family == 'K':
                prefix = b'CONFIG_'

            for tok in tokens:
                even = not even
                tok2 = prefix + tok
                if (even and db.defs.exists(tok2) and
                    (lib.compatibleFamily(db.defs.get(tok2).get_families(), family) or
                    lib.compatibleMacro(db.defs.get(tok2).get_macros(), family))):
                    tok = b'\033[31m' + tok2 + b'\033[0m'
                else:
                    tok = lib.unescape(tok)
                buffer.write(tok)
            return decode(buffer.getvalue())
        else:
            return decode(script('get-file', version, path))

    elif cmd == 'family':
        # Get the family of a given file

        filename = args[0]

        return lib.getFileFamily(filename)

    elif cmd == 'dts-comp':
        # Get state of dts_comp_support

        return dts_comp_support

    elif cmd == 'dts-comp-exists':
        # Check if a dts compatible string exists

        ident = args[0]
        if dts_comp_support:
            return db.comps.exists(ident)
        else:
            return False

    elif cmd == 'keys':
        # Return all keys of a given database
        # /!\ This can take a while /!\

        name = args[0]

        if name == 'vars':
            return db.vars.get_keys()
        elif name == 'blob':
            return db.blob.get_keys()
        elif name == 'hash':
            return db.hash.get_keys()
        elif name == 'file':
            return db.file.get_keys()
        elif name == 'vers':
            return db.vers.get_keys()
        elif name == 'defs':
            return db.defs.get_keys()
        elif name == 'refs':
            return db.refs.get_keys()
        elif name == 'docs':
            return db.docs.get_keys()
        elif name == 'comps' and dts_comp_support:
            return db.comps.get_keys()
        elif name == 'comps_docs' and dts_comp_support:
            return db.comps_docs.get_keys()
        else:
            return []

    elif cmd == 'ident':

        # Returns identifier search results

        version = args[0]
        ident = args[1]
        family = args[2]

        # DT bindings compatible strings are handled differently
        if family == 'B':
            return get_idents_comps(version, ident)
        else:
            return get_idents_defs(version, ident, family)

    else:
        return('Unknown subcommand: ' + cmd + '\n')
Esempio n. 2
0
# Throughout, an "idx" is the sequential number associated with a blob.
# This is different from that blob's Git hash.

from sys import argv
from threading import Thread, Lock, Event, Condition

import lib
from lib import script, scriptLines
import data
from data import PathList
from find_compatible_dts import FindCompatibleDTS

verbose = False

dts_comp_support = int(script('dts-comp'))

compatibles_parser = FindCompatibleDTS()

db = data.DB(lib.getDataDir(), readonly=False, dtscomp=dts_comp_support)

# Number of cpu threads (+2 for version indexing)
cpu = 10
threads_list = []

hash_file_lock = Lock()  # Lock for db.hash and db.file
blobs_lock = Lock()  # Lock for db.blobs
defs_lock = Lock()  # Lock for db.defs
refs_lock = Lock()  # Lock for db.refs
docs_lock = Lock()  # Lock for db.docs
comps_lock = Lock()  # Lock for db.comps
Esempio n. 3
0
def query (cmd, *args):
    buffer = BytesIO()

    def echo (arg):
        buffer.write (arg)

    if cmd == 'versions':
        for p in scriptLines ('list-tags', '-h'):
            if db.vers.exists (p.split(b' ')[2]):
                echo (p + b'\n')

    elif cmd == 'latest':
        p = script ('get-latest')
        echo (p)

    elif cmd == 'type':
        version = args[0]
        path = args[1]
        p = script ('get-type', version, path)
        echo (p)

    elif cmd == 'dir':
        version = args[0]
        path = args[1]
        p = script ('get-dir', version, path)
        echo (p)

    elif cmd == 'file':
        version = args[0]
        path = args[1]
        ext = os.path.splitext(path)[1]

        if ext in ['.c', '.cc', '.cpp', '.h']:
            tokens = scriptLines ('tokenize-file', version, path)
            even = True
            for tok in tokens:
                even = not even
                if even and db.defs.exists (tok) and lib.isIdent (tok):
                    tok = b'\033[31m' + tok + b'\033[0m'
                else:
                    tok = lib.unescape (tok)
                echo (tok)
        else:
            p = script ('get-file', version, path)
            echo (p)

    elif cmd == 'ident':
        version = args[0]
        ident = args[1]

        if not db.defs.exists (ident):
            echo (('Unknown identifier: ' + ident + '\n').encode())
            return buffer.getvalue()

        if not db.vers.exists (version):
            echo (('Unknown version: ' + version + '\n').encode())
            return buffer.getvalue()

        vers = db.vers.get (version).iter()
        defs = db.defs.get (ident).iter (dummy=True)
        # FIXME: see why we can have a discrepancy between defs and refs
        if db.refs.exists (ident):
            refs = db.refs.get (ident).iter (dummy=True)
        else:
            refs = data.RefList().iter (dummy=True)

        id2, type, dline = next (defs)
        id3, rlines = next (refs)

        dBuf = []
        rBuf = []

        for id1, path in vers:
            while id1 > id2:
                id2, type, dline = next (defs)
            while id1 > id3:
                id3, rlines = next (refs)
            while id1 == id2:
                dBuf.append ((path, type, dline))
                id2, type, dline = next (defs)
            if id1 == id3:
                rBuf.append ((path, rlines))

        echo (('Defined in ' + str(len(dBuf)) + ' files:\n').encode())
        for path, type, dline in sorted (dBuf):
            echo ((path + ': ' + str (dline) + ' (' + type + ')\n').encode())

        echo (('\nReferenced in ' + str(len(rBuf)) + ' files:\n').encode())
        for path, rlines in sorted (rBuf):
            echo ((path + ': ' + rlines + '\n').encode())

    else:
        echo (('Unknown subcommand: ' + cmd + '\n').encode())

    return buffer.getvalue()
Esempio n. 4
0
def query(cmd, *args):
    if cmd == 'versions':

        # Returns the list of indexed versions in the following format:
        # topmenu submenu tag
        # Example: v3 v3.1 v3.1-rc10
        versions = OrderedDict()

        for line in scriptLines('list-tags', '-h'):
            taginfo = decode(line).split(' ')
            num = len(taginfo)
            topmenu, submenu = 'FIXME', 'FIXME'

            if (num == 1):
                tag, = taginfo
            elif (num == 2):
                submenu, tag = taginfo
            elif (num == 3):
                topmenu, submenu, tag = taginfo

            if db.vers.exists(tag):
                if topmenu not in versions:
                    versions[topmenu] = OrderedDict()
                if submenu not in versions[topmenu]:
                    versions[topmenu][submenu] = []
                versions[topmenu][submenu].append(tag)

        return versions

    elif cmd == 'latest':

        # Returns the tag considered as the latest one
        previous = None
        tag = ''
        index = 0

        # If we get the same tag twice, we are at the oldest one
        while not db.vers.exists(tag) and previous != tag:
            previous = tag
            tag = decode(script('get-latest', str(index))).rstrip('\n')
            index += 1

        return tag

    elif cmd == 'type':

        # Returns the type (blob or tree) associated to
        # the given path. Example:
        # > ./query.py type v3.1-rc10 /Makefile
        # blob
        # > ./query.py type v3.1-rc10 /arch
        # tree

        version = args[0]
        path = args[1]
        return decode(script('get-type', version, path)).strip()

    elif cmd == 'exist':

        # Returns True if the requested file exists, overwise returns False

        version = args[0]
        path = args[1]

        dirname, filename = os.path.split(path)

        entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
        for entry in entries:
            fname = entry.split(" ")[1]

            if fname == filename:
                return True

        return False

    elif cmd == 'dir':

        # Returns the contents (trees or blobs) of the specified directory
        # Example: ./query.py dir v3.1-rc10 /arch

        version = args[0]
        path = args[1]
        entries_str = decode(script('get-dir', version, path))
        return entries_str.split("\n")[:-1]

    elif cmd == 'file':

        # Returns the contents of the specified file
        # Tokens are marked for further processing
        # Example: ./query.py file v3.1-rc10 /Makefile

        version = args[0]
        path = args[1]

        filename = os.path.basename(path)
        family = lib.getFileFamily(filename)

        if family != None:
            buffer = BytesIO()
            tokens = scriptLines('tokenize-file', version, path, family)
            even = True

            prefix = b''
            if family == 'K':
                prefix = b'CONFIG_'

            for tok in tokens:
                even = not even
                tok2 = prefix + tok
                if (even and db.defs.exists(tok2) and lib.isIdent(tok2)
                        and lib.compatibleFamily(
                            db.defs.get(tok2).get_families(), family)):
                    tok = b'\033[31m' + tok2 + b'\033[0m'
                else:
                    tok = lib.unescape(tok)
                buffer.write(tok)
            return decode(buffer.getvalue())
        else:
            return decode(script('get-file', version, path))

    elif cmd == 'family':
        # Get the family of a given file

        filename = args[0]

        return lib.getFileFamily(filename)

    elif cmd == 'ident':

        # Returns identifier search results

        version = args[0]
        ident = args[1]
        family = args[2]

        symbol_definitions = []
        symbol_references = []
        symbol_doccomments = []

        if not db.defs.exists(ident):
            return symbol_definitions, symbol_references, symbol_doccomments

        if not db.vers.exists(version):
            return symbol_definitions, symbol_references, symbol_doccomments

        files_this_version = db.vers.get(version).iter()
        defs_this_ident = db.defs.get(ident).iter(dummy=True)
        # FIXME: see why we can have a discrepancy between defs_this_ident and refs
        if db.refs.exists(ident):
            refs = db.refs.get(ident).iter(dummy=True)
        else:
            refs = data.RefList().iter(dummy=True)

        if db.docs.exists(ident):
            docs = db.docs.get(ident).iter(dummy=True)
        else:
            docs = data.RefList().iter(dummy=True)

        # vers, defs, refs, and docs are all populated by update.py in order of
        # idx, and there is a one-to-one mapping between blob hashes and idx
        # values.  Therefore, we can sequentially step through the defs, refs,
        # and docs for each file in a version.

        def_idx, def_type, def_line, def_family = next(defs_this_ident)
        ref_idx, ref_lines, ref_family = next(refs)
        doc_idx, doc_line, doc_family = next(docs)

        dBuf = []
        rBuf = []
        docBuf = []

        for file_idx, file_path in files_this_version:
            # Advance defs, refs, and docs to the current file
            while def_idx < file_idx:
                def_idx, def_type, def_line, def_family = next(defs_this_ident)
            while ref_idx < file_idx:
                ref_idx, ref_lines, ref_family = next(refs)
            while doc_idx < file_idx:
                doc_idx, doc_line, doc_family = next(docs)

            # Copy information about this identifier into dBuf, rBuf, and docBuf.
            while def_idx == file_idx:
                if def_family == family or family == 'A':
                    dBuf.append((file_path, def_type, def_line))
                def_idx, def_type, def_line, def_family = next(defs_this_ident)

            if ref_idx == file_idx:
                if lib.compatibleFamily(family, ref_family) or family == 'A':
                    rBuf.append((file_path, ref_lines))

            if doc_idx == file_idx:  # TODO should this be a `while`?
                docBuf.append((file_path, doc_line))

        for path, type, dline in sorted(dBuf):
            symbol_definitions.append(SymbolInstance(path, dline, type))

        for path, rlines in sorted(rBuf):
            symbol_references.append(SymbolInstance(path, rlines))

        for path, docline in sorted(docBuf):
            symbol_doccomments.append(SymbolInstance(path, docline))

        return symbol_definitions, symbol_references, symbol_doccomments

    else:
        return ('Unknown subcommand: ' + cmd + '\n')
Esempio n. 5
0
    print(argv[0] + ': LXR_DATA_DIR needs to be set')
    exit(1)

db = data.DB(dbDir, readonly=True)

cmd = argv[1]

if cmd == 'versions':
    for p in scriptLines('list-tags', '-r'):
        if db.vers.exists(p):
            echo(p + b'\n')

elif cmd == 'type':
    version = argv[2]
    path = argv[3]
    p = script('get-type', version, path)
    echo(p)

elif cmd == 'dir':
    version = argv[2]
    path = argv[3]
    p = script('get-dir', version, path)
    echo(p)

elif cmd == 'file':
    version = argv[2]
    path = argv[3]
    ext = path[-2:]

    if ext == '.c' or ext == '.h':
        tokens = scriptLines('tokenize-file', version, path)
Esempio n. 6
0
def query(cmd, *args):
    if cmd == 'versions':

        # Returns the list of indexed versions in the following format:
        # topmenu submenu tag
        # Example: v3 v3.1 v3.1-rc10
        versions = OrderedDict()

        for line in scriptLines('list-tags', '-h'):
            taginfo = decode(line).split(' ')
            num = len(taginfo)
            topmenu, submenu = 'FIXME', 'FIXME'

            if (num == 1):
                tag, = taginfo
            elif (num == 2):
                submenu, tag = taginfo
            elif (num == 3):
                topmenu, submenu, tag = taginfo

            if db.vers.exists(tag):
                if topmenu not in versions:
                    versions[topmenu] = OrderedDict()
                if submenu not in versions[topmenu]:
                    versions[topmenu][submenu] = []
                versions[topmenu][submenu].append(tag)

        return versions

    elif cmd == 'latest':

        # Returns the tag considered as the latest one
        # TODO: this latest tag may have just been retrieved
        # in the git repository and may not have been indexed yet
        # This could results in failed queries

        return decode(script('get-latest')).rstrip('\n')

    elif cmd == 'type':

        # Returns the type (blob or tree) associated to
        # the given path. Example:
        # > ./query.py type v3.1-rc10 /Makefile
        # blob
        # > ./query.py type v3.1-rc10 /arch
        # tree

        version = args[0]
        path = args[1]
        return decode(script('get-type', version, path)).strip()

    elif cmd == 'dir':

        # Returns the contents (trees or blobs) of the specified directory
        # Example: ./query.py dir v3.1-rc10 /arch

        version = args[0]
        path = args[1]
        entries_str = decode(script('get-dir', version, path))
        return entries_str.split("\n")[:-1]

    elif cmd == 'file':

        # Returns the contents of the specified file
        # Tokens are marked for further processing
        # Example: ./query.py file v3.1-rc10 /Makefile

        version = args[0]
        path = args[1]

        if lib.hasSupportedExt(path):
            buffer = BytesIO()
            tokens = scriptLines('tokenize-file', version, path)
            even = True
            for tok in tokens:
                even = not even
                if even and db.defs.exists(tok) and lib.isIdent(tok):
                    tok = b'\033[31m' + tok + b'\033[0m'
                else:
                    tok = lib.unescape(tok)
                buffer.write(tok)
            return decode(buffer.getvalue())
        else:
            return decode(script('get-file', version, path))

    elif cmd == 'ident':

        # Returns identifier search results

        version = args[0]
        ident = args[1]

        symbol_definitions = []
        symbol_references = []
        symbol_doccomments = []

        if not db.defs.exists(ident):
            return symbol_definitions, symbol_references, symbol_doccomments

        if not db.vers.exists(version):
            return symbol_definitions, symbol_references, symbol_doccomments

        files_this_version = db.vers.get(version).iter()
        defs_this_ident = db.defs.get(ident).iter(dummy=True)
        # FIXME: see why we can have a discrepancy between defs_this_ident and refs
        if db.refs.exists(ident):
            refs = db.refs.get(ident).iter(dummy=True)
        else:
            refs = data.RefList().iter(dummy=True)

        if db.docs.exists(ident):
            docs = db.docs.get(ident).iter(dummy=True)
        else:
            docs = data.RefList().iter(dummy=True)

        # vers, defs, refs, and docs are all populated by update.py in order of
        # idx, and there is a one-to-one mapping between blob hashes and idx
        # values.  Therefore, we can sequentially step through the defs, refs,
        # and docs for each file in a version.

        def_idx, def_type, def_line = next(defs_this_ident)
        ref_idx, ref_lines = next(refs)
        doc_idx, doc_line = next(docs)

        dBuf = []
        rBuf = []
        docBuf = []

        for file_idx, file_path in files_this_version:
            # Advance defs, refs, and docs to the current file
            while def_idx < file_idx:
                def_idx, def_type, def_line = next(defs_this_ident)
            while ref_idx < file_idx:
                ref_idx, ref_lines = next(refs)
            while doc_idx < file_idx:
                doc_idx, doc_line = next(docs)

            # Copy information about this identifier into dBuf, rBuf, and docBuf.
            while def_idx == file_idx:
                dBuf.append((file_path, def_type, def_line))
                def_idx, def_type, def_line = next(defs_this_ident)

            if ref_idx == file_idx:
                rBuf.append((file_path, ref_lines))

            if doc_idx == file_idx:  # TODO should this be a `while`?
                docBuf.append((file_path, doc_line))

        for path, type, dline in sorted(dBuf):
            symbol_definitions.append(SymbolInstance(path, dline, type))

        for path, rlines in sorted(rBuf):
            symbol_references.append(SymbolInstance(path, rlines))

        for path, docline in sorted(docBuf):
            symbol_doccomments.append(SymbolInstance(path, docline))

        return symbol_definitions, symbol_references, symbol_doccomments

    else:
        return ('Unknown subcommand: ' + cmd + '\n')
Esempio n. 7
0
def query(cmd, *args):
    if cmd == 'versions':

        # Returns the list of indexed versions in the following format:
        # topmenu submenu tag
        # Example: v3 v3.1 v3.1-rc10
        versions = OrderedDict()

        for line in scriptLines('list-tags', '-h'):
            taginfo = decode(line).split(' ')
            num = len(taginfo)
            topmenu, submenu = 'FIXME', 'FIXME'

            if (num == 1):
                tag, = taginfo
            elif (num == 2):
                submenu, tag = taginfo
            elif (num == 3):
                topmenu, submenu, tag = taginfo

            if db.vers.exists(tag):
                if topmenu not in versions:
                    versions[topmenu] = OrderedDict()
                if submenu not in versions[topmenu]:
                    versions[topmenu][submenu] = []
                versions[topmenu][submenu].append(tag)

        return versions

    elif cmd == 'latest':

        # Returns the tag considered as the latest one
        # TODO: this latest tag may have just been retrieved
        # in the git repository and may not have been indexed yet
        # This could results in failed queries

        return decode(script('get-latest')).rstrip('\n')

    elif cmd == 'type':

        # Returns the type (blob or tree) associated to
        # the given path. Example:
        # > ./query.py type v3.1-rc10 /Makefile
        # blob
        # > ./query.py type v3.1-rc10 /arch
        # tree

        version = args[0]
        path = args[1]
        return decode(script('get-type', version, path)).strip()

    elif cmd == 'dir':

        # Returns the contents (trees or blobs) of the specified directory
        # Example: ./query.py dir v3.1-rc10 /arch

        version = args[0]
        path = args[1]
        entries_str = decode(script('get-dir', version, path))
        return entries_str.split("\n")[:-1]

    elif cmd == 'file':

        # Returns the contents of the specified file
        # Tokens are marked for further processing
        # Example: ./query.py file v3.1-rc10 /Makefile

        version = args[0]
        path = args[1]

        if lib.hasSupportedExt(path):
            buffer = BytesIO()
            tokens = scriptLines('tokenize-file', version, path)
            even = True
            for tok in tokens:
                even = not even
                if even and db.defs.exists(tok) and lib.isIdent(tok):
                    tok = b'\033[31m' + tok + b'\033[0m'
                else:
                    tok = lib.unescape(tok)
                buffer.write(tok)
            return decode(buffer.getvalue())
        else:
            return decode(script('get-file', version, path))

    elif cmd == 'ident':

        # Returns identifier search results

        version = args[0]
        ident = args[1]

        symbol_definitions = []
        symbol_references = []

        if not db.defs.exists(ident):
            return symbol_definitions, symbol_references

        if not db.vers.exists(version):
            return symbol_definitions, symbol_references

        vers = db.vers.get(version).iter()
        defs = db.defs.get(ident).iter(dummy=True)
        # FIXME: see why we can have a discrepancy between defs and refs
        if db.refs.exists(ident):
            refs = db.refs.get(ident).iter(dummy=True)
        else:
            refs = data.RefList().iter(dummy=True)

        id2, type, dline = next(defs)
        id3, rlines = next(refs)

        dBuf = []
        rBuf = []

        for id1, path in vers:
            while id1 > id2:
                id2, type, dline = next(defs)
            while id1 > id3:
                id3, rlines = next(refs)
            while id1 == id2:
                dBuf.append((path, type, dline))
                id2, type, dline = next(defs)
            if id1 == id3:
                rBuf.append((path, rlines))

        for path, type, dline in sorted(dBuf):
            symbol_definitions.append(SymbolInstance(path, dline, type))

        for path, rlines in sorted(rBuf):
            symbol_references.append(SymbolInstance(path, rlines))

        return symbol_definitions, symbol_references

    else:
        return ('Unknown subcommand: ' + cmd + '\n')