def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu,tag = taginfo
elif (num ==3):
topmenu,submenu,tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and
(lib.compatibleFamily(db.defs.get(tok2).get_families(), family) or
lib.compatibleMacro(db.defs.get(tok2).get_macros(), family))):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'dts-comp':
# Get state of dts_comp_support
return dts_comp_support
elif cmd == 'dts-comp-exists':
# Check if a dts compatible string exists
ident = args[0]
if dts_comp_support:
return db.comps.exists(ident)
else:
return False
elif cmd == 'keys':
# Return all keys of a given database
# /!\ This can take a while /!\
name = args[0]
if name == 'vars':
return db.vars.get_keys()
elif name == 'blob':
return db.blob.get_keys()
elif name == 'hash':
return db.hash.get_keys()
elif name == 'file':
return db.file.get_keys()
elif name == 'vers':
return db.vers.get_keys()
elif name == 'defs':
return db.defs.get_keys()
elif name == 'refs':
return db.refs.get_keys()
elif name == 'docs':
return db.docs.get_keys()
elif name == 'comps' and dts_comp_support:
return db.comps.get_keys()
elif name == 'comps_docs' and dts_comp_support:
return db.comps_docs.get_keys()
else:
return []
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
# DT bindings compatible strings are handled differently
if family == 'B':
return get_idents_comps(version, ident)
else:
return get_idents_defs(version, ident, family)
else:
return('Unknown subcommand: ' + cmd + '\n')
# Throughout, an "idx" is the sequential number associated with a blob.
# This is different from that blob's Git hash.
from sys import argv
from threading import Thread, Lock, Event, Condition
import lib
from lib import script, scriptLines
import data
from data import PathList
from find_compatible_dts import FindCompatibleDTS
verbose = False
dts_comp_support = int(script('dts-comp'))
compatibles_parser = FindCompatibleDTS()
db = data.DB(lib.getDataDir(), readonly=False, dtscomp=dts_comp_support)
# Number of cpu threads (+2 for version indexing)
cpu = 10
threads_list = []
hash_file_lock = Lock() # Lock for db.hash and db.file
blobs_lock = Lock() # Lock for db.blobs
defs_lock = Lock() # Lock for db.defs
refs_lock = Lock() # Lock for db.refs
docs_lock = Lock() # Lock for db.docs
comps_lock = Lock() # Lock for db.comps
def query (cmd, *args):
buffer = BytesIO()
def echo (arg):
buffer.write (arg)
if cmd == 'versions':
for p in scriptLines ('list-tags', '-h'):
if db.vers.exists (p.split(b' ')[2]):
echo (p + b'\n')
elif cmd == 'latest':
p = script ('get-latest')
echo (p)
elif cmd == 'type':
version = args[0]
path = args[1]
p = script ('get-type', version, path)
echo (p)
elif cmd == 'dir':
version = args[0]
path = args[1]
p = script ('get-dir', version, path)
echo (p)
elif cmd == 'file':
version = args[0]
path = args[1]
ext = os.path.splitext(path)[1]
if ext in ['.c', '.cc', '.cpp', '.h']:
tokens = scriptLines ('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists (tok) and lib.isIdent (tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape (tok)
echo (tok)
else:
p = script ('get-file', version, path)
echo (p)
elif cmd == 'ident':
version = args[0]
ident = args[1]
if not db.defs.exists (ident):
echo (('Unknown identifier: ' + ident + '\n').encode())
return buffer.getvalue()
if not db.vers.exists (version):
echo (('Unknown version: ' + version + '\n').encode())
return buffer.getvalue()
vers = db.vers.get (version).iter()
defs = db.defs.get (ident).iter (dummy=True)
# FIXME: see why we can have a discrepancy between defs and refs
if db.refs.exists (ident):
refs = db.refs.get (ident).iter (dummy=True)
else:
refs = data.RefList().iter (dummy=True)
id2, type, dline = next (defs)
id3, rlines = next (refs)
dBuf = []
rBuf = []
for id1, path in vers:
while id1 > id2:
id2, type, dline = next (defs)
while id1 > id3:
id3, rlines = next (refs)
while id1 == id2:
dBuf.append ((path, type, dline))
id2, type, dline = next (defs)
if id1 == id3:
rBuf.append ((path, rlines))
echo (('Defined in ' + str(len(dBuf)) + ' files:\n').encode())
for path, type, dline in sorted (dBuf):
echo ((path + ': ' + str (dline) + ' (' + type + ')\n').encode())
echo (('\nReferenced in ' + str(len(rBuf)) + ' files:\n').encode())
for path, rlines in sorted (rBuf):
echo ((path + ': ' + rlines + '\n').encode())
else:
echo (('Unknown subcommand: ' + cmd + '\n').encode())
return buffer.getvalue()
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and lib.isIdent(tok2)
and lib.compatibleFamily(
db.defs.get(tok2).get_families(), family)):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line, def_family = next(defs_this_ident)
ref_idx, ref_lines, ref_family = next(refs)
doc_idx, doc_line, doc_family = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line, def_family = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines, ref_family = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line, doc_family = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
if def_family == family or family == 'A':
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line, def_family = next(defs_this_ident)
if ref_idx == file_idx:
if lib.compatibleFamily(family, ref_family) or family == 'A':
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
print(argv[0] + ': LXR_DATA_DIR needs to be set')
exit(1)
db = data.DB(dbDir, readonly=True)
cmd = argv[1]
if cmd == 'versions':
for p in scriptLines('list-tags', '-r'):
if db.vers.exists(p):
echo(p + b'\n')
elif cmd == 'type':
version = argv[2]
path = argv[3]
p = script('get-type', version, path)
echo(p)
elif cmd == 'dir':
version = argv[2]
path = argv[3]
p = script('get-dir', version, path)
echo(p)
elif cmd == 'file':
version = argv[2]
path = argv[3]
ext = path[-2:]
if ext == '.c' or ext == '.h':
tokens = scriptLines('tokenize-file', version, path)
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
# TODO: this latest tag may have just been retrieved
# in the git repository and may not have been indexed yet
# This could results in failed queries
return decode(script('get-latest')).rstrip('\n')
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
if lib.hasSupportedExt(path):
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line = next(defs_this_ident)
ref_idx, ref_lines = next(refs)
doc_idx, doc_line = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line = next(defs_this_ident)
if ref_idx == file_idx:
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
# TODO: this latest tag may have just been retrieved
# in the git repository and may not have been indexed yet
# This could results in failed queries
return decode(script('get-latest')).rstrip('\n')
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
if lib.hasSupportedExt(path):
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
symbol_definitions = []
symbol_references = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references
if not db.vers.exists(version):
return symbol_definitions, symbol_references
vers = db.vers.get(version).iter()
defs = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
id2, type, dline = next(defs)
id3, rlines = next(refs)
dBuf = []
rBuf = []
for id1, path in vers:
while id1 > id2:
id2, type, dline = next(defs)
while id1 > id3:
id3, rlines = next(refs)
while id1 == id2:
dBuf.append((path, type, dline))
id2, type, dline = next(defs)
if id1 == id3:
rBuf.append((path, rlines))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
return symbol_definitions, symbol_references
else:
return ('Unknown subcommand: ' + cmd + '\n')