def updateReferences(blobs):
for blob in blobs:
if (blob % 100 == 0): print('R:', blob)
hash = db.hash.get(blob)
filename = db.file.get(blob)
ext = filename[-2:]
if not (ext == '.c' or ext == '.h'): continue
tokens = scriptLines('tokenize-file', '-b', hash)
even = True
lineNum = 1
idents = {}
for tok in tokens:
even = not even
if even:
if db.defs.exists(tok) and lib.isIdent(tok):
if tok in idents:
idents[tok] += ',' + str(lineNum)
else:
idents[tok] = str(lineNum)
else:
lineNum += tok.count(b'\1')
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(blob, lines)
db.refs.put(ident, obj)
def updateReferences(idxes):
for idx in idxes:
if (idx % 1000 == 0): progress('refs: ' + str(idx))
hash = db.hash.get(idx)
filename = db.file.get(idx)
if not lib.hasSupportedExt(filename): continue
tokens = scriptLines('tokenize-file', '-b', hash)
even = True
lineNum = 1
idents = {}
for tok in tokens:
even = not even
if even:
if db.defs.exists(tok) and lib.isIdent(tok):
if tok in idents:
idents[tok] += ',' + str(lineNum)
else:
idents[tok] = str(lineNum)
else:
lineNum += tok.count(b'\1')
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines)
if verbose:
print(f"ref: {ident} in #{idx} @ {lines}")
db.refs.put(ident, obj)
def update_definitions(self, idxes):
global hash_file_lock, defs_lock, tags_defs
for idx in idxes:
if (idx % 1000 == 0): progress('defs: ' + str(idx), tags_defs[0])
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family in [None, 'M']: continue
lines = scriptLines('parse-defs', hash, filename, family)
with defs_lock:
for l in lines:
ident, type, line = l.split(b' ')
type = type.decode()
line = int(line.decode())
if db.defs.exists(ident):
obj = db.defs.get(ident)
elif lib.isIdent(ident):
obj = data.DefList()
else:
continue
obj.append(idx, type, line, family)
if verbose:
print(f"def {type} {ident} in #{idx} @ {line}")
db.defs.put(ident, obj)
def update_references(self, idxes):
global hash_file_lock, defs_lock
for idx in idxes:
if (idx % 1000 == 0): progress('refs: ' + str(idx), self.index + 1)
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
prefix = b''
# Kconfig values are saved as CONFIG_<value>
if family == 'K':
prefix = b'CONFIG_'
tokens = scriptLines('tokenize-file', '-b', hash, family)
even = True
line_num = 1
idents = {}
for tok in tokens:
even = not even
if even:
tok = prefix + tok
with defs_lock:
if db.defs.exists(tok) and lib.isIdent(tok):
if tok in idents:
idents[tok] += ',' + str(line_num)
else:
idents[tok] = str(line_num)
else:
line_num += tok.count(b'\1')
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines, family)
if verbose:
print(f"ref: {ident} in #{idx} @ {lines}")
db.refs.put(ident, obj)
def query (cmd, *args):
buffer = BytesIO()
def echo (arg):
buffer.write (arg)
if cmd == 'versions':
for p in scriptLines ('list-tags', '-h'):
if db.vers.exists (p.split(b' ')[2]):
echo (p + b'\n')
elif cmd == 'latest':
p = script ('get-latest')
echo (p)
elif cmd == 'type':
version = args[0]
path = args[1]
p = script ('get-type', version, path)
echo (p)
elif cmd == 'dir':
version = args[0]
path = args[1]
p = script ('get-dir', version, path)
echo (p)
elif cmd == 'file':
version = args[0]
path = args[1]
ext = os.path.splitext(path)[1]
if ext in ['.c', '.cc', '.cpp', '.h']:
tokens = scriptLines ('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists (tok) and lib.isIdent (tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape (tok)
echo (tok)
else:
p = script ('get-file', version, path)
echo (p)
elif cmd == 'ident':
version = args[0]
ident = args[1]
if not db.defs.exists (ident):
echo (('Unknown identifier: ' + ident + '\n').encode())
return buffer.getvalue()
if not db.vers.exists (version):
echo (('Unknown version: ' + version + '\n').encode())
return buffer.getvalue()
vers = db.vers.get (version).iter()
defs = db.defs.get (ident).iter (dummy=True)
# FIXME: see why we can have a discrepancy between defs and refs
if db.refs.exists (ident):
refs = db.refs.get (ident).iter (dummy=True)
else:
refs = data.RefList().iter (dummy=True)
id2, type, dline = next (defs)
id3, rlines = next (refs)
dBuf = []
rBuf = []
for id1, path in vers:
while id1 > id2:
id2, type, dline = next (defs)
while id1 > id3:
id3, rlines = next (refs)
while id1 == id2:
dBuf.append ((path, type, dline))
id2, type, dline = next (defs)
if id1 == id3:
rBuf.append ((path, rlines))
echo (('Defined in ' + str(len(dBuf)) + ' files:\n').encode())
for path, type, dline in sorted (dBuf):
echo ((path + ': ' + str (dline) + ' (' + type + ')\n').encode())
echo (('\nReferenced in ' + str(len(rBuf)) + ' files:\n').encode())
for path, rlines in sorted (rBuf):
echo ((path + ': ' + rlines + '\n').encode())
else:
echo (('Unknown subcommand: ' + cmd + '\n').encode())
return buffer.getvalue()
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and lib.isIdent(tok2)
and lib.compatibleFamily(
db.defs.get(tok2).get_families(), family)):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line, def_family = next(defs_this_ident)
ref_idx, ref_lines, ref_family = next(refs)
doc_idx, doc_line, doc_family = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line, def_family = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines, ref_family = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line, doc_family = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
if def_family == family or family == 'A':
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line, def_family = next(defs_this_ident)
if ref_idx == file_idx:
if lib.compatibleFamily(family, ref_family) or family == 'A':
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
version = argv[2]
path = argv[3]
p = script('get-dir', version, path)
echo(p)
elif cmd == 'file':
version = argv[2]
path = argv[3]
ext = path[-2:]
if ext == '.c' or ext == '.h':
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
echo(tok)
else:
p = script('get-file', version, path)
echo(p)
elif cmd == 'ident':
version = argv[2]
ident = argv[3]
if not db.defs.exists(ident):
print(argv[0] + ': Unknown identifier: ' + ident)
exit()
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
# TODO: this latest tag may have just been retrieved
# in the git repository and may not have been indexed yet
# This could results in failed queries
return decode(script('get-latest')).rstrip('\n')
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
if lib.hasSupportedExt(path):
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line = next(defs_this_ident)
ref_idx, ref_lines = next(refs)
doc_idx, doc_line = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line = next(defs_this_ident)
if ref_idx == file_idx:
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
# TODO: this latest tag may have just been retrieved
# in the git repository and may not have been indexed yet
# This could results in failed queries
return decode(script('get-latest')).rstrip('\n')
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
if lib.hasSupportedExt(path):
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
symbol_definitions = []
symbol_references = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references
if not db.vers.exists(version):
return symbol_definitions, symbol_references
vers = db.vers.get(version).iter()
defs = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
id2, type, dline = next(defs)
id3, rlines = next(refs)
dBuf = []
rBuf = []
for id1, path in vers:
while id1 > id2:
id2, type, dline = next(defs)
while id1 > id3:
id3, rlines = next(refs)
while id1 == id2:
dBuf.append((path, type, dline))
id2, type, dline = next(defs)
if id1 == id3:
rBuf.append((path, rlines))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
return symbol_definitions, symbol_references
else:
return ('Unknown subcommand: ' + cmd + '\n')
