def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu,tag = taginfo
elif (num ==3):
topmenu,submenu,tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and
(lib.compatibleFamily(db.defs.get(tok2).get_families(), family) or
lib.compatibleMacro(db.defs.get(tok2).get_macros(), family))):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'dts-comp':
# Get state of dts_comp_support
return dts_comp_support
elif cmd == 'dts-comp-exists':
# Check if a dts compatible string exists
ident = args[0]
if dts_comp_support:
return db.comps.exists(ident)
else:
return False
elif cmd == 'keys':
# Return all keys of a given database
# /!\ This can take a while /!\
name = args[0]
if name == 'vars':
return db.vars.get_keys()
elif name == 'blob':
return db.blob.get_keys()
elif name == 'hash':
return db.hash.get_keys()
elif name == 'file':
return db.file.get_keys()
elif name == 'vers':
return db.vers.get_keys()
elif name == 'defs':
return db.defs.get_keys()
elif name == 'refs':
return db.refs.get_keys()
elif name == 'docs':
return db.docs.get_keys()
elif name == 'comps' and dts_comp_support:
return db.comps.get_keys()
elif name == 'comps_docs' and dts_comp_support:
return db.comps_docs.get_keys()
else:
return []
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
# DT bindings compatible strings are handled differently
if family == 'B':
return get_idents_comps(version, ident)
else:
return get_idents_defs(version, ident, family)
else:
return('Unknown subcommand: ' + cmd + '\n')
def query (cmd, *args):
buffer = BytesIO()
def echo (arg):
buffer.write (arg)
if cmd == 'versions':
for p in scriptLines ('list-tags', '-h'):
if db.vers.exists (p.split(b' ')[2]):
echo (p + b'\n')
elif cmd == 'latest':
p = script ('get-latest')
echo (p)
elif cmd == 'type':
version = args[0]
path = args[1]
p = script ('get-type', version, path)
echo (p)
elif cmd == 'dir':
version = args[0]
path = args[1]
p = script ('get-dir', version, path)
echo (p)
elif cmd == 'file':
version = args[0]
path = args[1]
ext = os.path.splitext(path)[1]
if ext in ['.c', '.cc', '.cpp', '.h']:
tokens = scriptLines ('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists (tok) and lib.isIdent (tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape (tok)
echo (tok)
else:
p = script ('get-file', version, path)
echo (p)
elif cmd == 'ident':
version = args[0]
ident = args[1]
if not db.defs.exists (ident):
echo (('Unknown identifier: ' + ident + '\n').encode())
return buffer.getvalue()
if not db.vers.exists (version):
echo (('Unknown version: ' + version + '\n').encode())
return buffer.getvalue()
vers = db.vers.get (version).iter()
defs = db.defs.get (ident).iter (dummy=True)
# FIXME: see why we can have a discrepancy between defs and refs
if db.refs.exists (ident):
refs = db.refs.get (ident).iter (dummy=True)
else:
refs = data.RefList().iter (dummy=True)
id2, type, dline = next (defs)
id3, rlines = next (refs)
dBuf = []
rBuf = []
for id1, path in vers:
while id1 > id2:
id2, type, dline = next (defs)
while id1 > id3:
id3, rlines = next (refs)
while id1 == id2:
dBuf.append ((path, type, dline))
id2, type, dline = next (defs)
if id1 == id3:
rBuf.append ((path, rlines))
echo (('Defined in ' + str(len(dBuf)) + ' files:\n').encode())
for path, type, dline in sorted (dBuf):
echo ((path + ': ' + str (dline) + ' (' + type + ')\n').encode())
echo (('\nReferenced in ' + str(len(rBuf)) + ' files:\n').encode())
for path, rlines in sorted (rBuf):
echo ((path + ': ' + rlines + '\n').encode())
else:
echo (('Unknown subcommand: ' + cmd + '\n').encode())
return buffer.getvalue()
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and lib.isIdent(tok2)
and lib.compatibleFamily(
db.defs.get(tok2).get_families(), family)):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line, def_family = next(defs_this_ident)
ref_idx, ref_lines, ref_family = next(refs)
doc_idx, doc_line, doc_family = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line, def_family = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines, ref_family = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line, doc_family = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
if def_family == family or family == 'A':
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line, def_family = next(defs_this_ident)
if ref_idx == file_idx:
if lib.compatibleFamily(family, ref_family) or family == 'A':
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
echo(p)
elif cmd == 'file':
version = argv[2]
path = argv[3]
ext = path[-2:]
if ext == '.c' or ext == '.h':
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
echo(tok)
else:
p = script('get-file', version, path)
echo(p)
elif cmd == 'ident':
version = argv[2]
ident = argv[3]
if not db.defs.exists(ident):
print(argv[0] + ': Unknown identifier: ' + ident)
exit()
if not db.vers.exists(version):
print(argv[0] + ': Unknown version: ' + version)
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
# TODO: this latest tag may have just been retrieved
# in the git repository and may not have been indexed yet
# This could results in failed queries
return decode(script('get-latest')).rstrip('\n')
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
if lib.hasSupportedExt(path):
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line = next(defs_this_ident)
ref_idx, ref_lines = next(refs)
doc_idx, doc_line = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line = next(defs_this_ident)
if ref_idx == file_idx:
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
# TODO: this latest tag may have just been retrieved
# in the git repository and may not have been indexed yet
# This could results in failed queries
return decode(script('get-latest')).rstrip('\n')
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
if lib.hasSupportedExt(path):
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
symbol_definitions = []
symbol_references = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references
if not db.vers.exists(version):
return symbol_definitions, symbol_references
vers = db.vers.get(version).iter()
defs = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
id2, type, dline = next(defs)
id3, rlines = next(refs)
dBuf = []
rBuf = []
for id1, path in vers:
while id1 > id2:
id2, type, dline = next(defs)
while id1 > id3:
id3, rlines = next(refs)
while id1 == id2:
dBuf.append((path, type, dline))
id2, type, dline = next(defs)
if id1 == id3:
rBuf.append((path, rlines))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
return symbol_definitions, symbol_references
else:
return ('Unknown subcommand: ' + cmd + '\n')