def update_doc_comments(self, idxes):
global hash_file_lock
for idx in idxes:
if (idx % 1000 == 0): progress('docs: ' + str(idx), self.index + 1)
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
lines = scriptLines('parse-docs', hash, filename)
for l in lines:
ident, line = l.split(b' ')
line = int(line.decode())
if db.docs.exists(ident):
obj = db.docs.get(ident)
else:
obj = data.RefList()
obj.append(idx, str(line), family)
if verbose:
print(f"doc: {ident} in #{idx} @ {line}")
db.docs.put(ident, obj)
def update_definitions(self, idxes):
global hash_file_lock, defs_lock
for idx in idxes:
if (idx % 1000 == 0): progress('defs: ' + str(idx), self.index + 1)
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
lines = scriptLines('parse-defs', hash, filename, family)
for l in lines:
ident, type, line = l.split(b' ')
type = type.decode()
line = int(line.decode())
with defs_lock:
if db.defs.exists(ident):
obj = db.defs.get(ident)
else:
obj = data.DefList()
obj.add_family(family)
obj.append(idx, type, line, family)
if verbose:
print(f"def {type} {ident} in #{idx} @ {line}")
with defs_lock:
db.defs.put(ident, obj)
def update_compatibles(self, idxes):
global hash_file_lock, comps_lock, tags_comps
for idx in idxes:
if (idx % 1000 == 0): progress('comps: ' + str(idx), tags_comps[0])
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family in [None, 'K', 'M']: continue
lines = compatibles_parser.run(scriptLines('get-blob', hash),
family)
comps = {}
for l in lines:
ident, line = l.split(' ')
if ident in comps:
comps[ident] += ',' + str(line)
else:
comps[ident] = str(line)
with comps_lock:
for ident, lines in comps.items():
if db.comps.exists(ident):
obj = db.comps.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines, family)
if verbose:
print(f"comps: {ident} in #{idx} @ {line}")
db.comps.put(ident, obj)
def update_definitions(self, idxes):
global hash_file_lock, defs_lock, tags_defs
for idx in idxes:
if (idx % 1000 == 0): progress('defs: ' + str(idx), tags_defs[0])
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family in [None, 'M']: continue
lines = scriptLines('parse-defs', hash, filename, family)
with defs_lock:
for l in lines:
ident, type, line = l.split(b' ')
type = type.decode()
line = int(line.decode())
defs_idxes[idx*idx_key_mod + line] = ident
if db.defs.exists(ident):
obj = db.defs.get(ident)
elif lib.isIdent(ident):
obj = data.DefList()
else:
continue
obj.append(idx, type, line, family)
if verbose:
print(f"def {type} {ident} in #{idx} @ {line}")
db.defs.put(ident, obj)
def update_references(self, idxes):
global hash_file_lock, defs_lock, refs_lock, tags_refs
for idx in idxes:
if (idx % 1000 == 0): progress('refs: ' + str(idx), tags_refs[0])
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
prefix = b''
# Kconfig values are saved as CONFIG_<value>
if family == 'K':
prefix = b'CONFIG_'
tokens = scriptLines('tokenize-file', '-b', hash, family)
even = True
line_num = 1
idents = {}
with defs_lock:
for tok in tokens:
even = not even
if even:
tok = prefix + tok
if (db.defs.exists(tok) and
not ( (idx*idx_key_mod + line_num) in defs_idxes and
defs_idxes[idx*idx_key_mod + line_num] == tok ) and
(family != 'M' or tok.startswith(b'CONFIG_'))):
# We only index CONFIG_??? in makefiles
if tok in idents:
idents[tok] += ',' + str(line_num)
else:
idents[tok] = str(line_num)
else:
line_num += tok.count(b'\1')
with refs_lock:
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines, family)
if verbose:
print(f"ref: {ident} in #{idx} @ {lines}")
db.refs.put(ident, obj)
def update_references(self, idxes):
global hash_file_lock, defs_lock
for idx in idxes:
if (idx % 1000 == 0): progress('refs: ' + str(idx), self.index + 1)
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
prefix = b''
# Kconfig values are saved as CONFIG_<value>
if family == 'K':
prefix = b'CONFIG_'
tokens = scriptLines('tokenize-file', '-b', hash, family)
even = True
line_num = 1
idents = {}
for tok in tokens:
even = not even
if even:
tok = prefix + tok
with defs_lock:
if db.defs.exists(tok) and lib.isIdent(tok):
if tok in idents:
idents[tok] += ',' + str(line_num)
else:
idents[tok] = str(line_num)
else:
line_num += tok.count(b'\1')
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines, family)
if verbose:
print(f"ref: {ident} in #{idx} @ {lines}")
db.refs.put(ident, obj)
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu,tag = taginfo
elif (num ==3):
topmenu,submenu,tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and
(lib.compatibleFamily(db.defs.get(tok2).get_families(), family) or
lib.compatibleMacro(db.defs.get(tok2).get_macros(), family))):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'dts-comp':
# Get state of dts_comp_support
return dts_comp_support
elif cmd == 'dts-comp-exists':
# Check if a dts compatible string exists
ident = args[0]
if dts_comp_support:
return db.comps.exists(ident)
else:
return False
elif cmd == 'keys':
# Return all keys of a given database
# /!\ This can take a while /!\
name = args[0]
if name == 'vars':
return db.vars.get_keys()
elif name == 'blob':
return db.blob.get_keys()
elif name == 'hash':
return db.hash.get_keys()
elif name == 'file':
return db.file.get_keys()
elif name == 'vers':
return db.vers.get_keys()
elif name == 'defs':
return db.defs.get_keys()
elif name == 'refs':
return db.refs.get_keys()
elif name == 'docs':
return db.docs.get_keys()
elif name == 'comps' and dts_comp_support:
return db.comps.get_keys()
elif name == 'comps_docs' and dts_comp_support:
return db.comps_docs.get_keys()
else:
return []
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
# DT bindings compatible strings are handled differently
if family == 'B':
return get_idents_comps(version, ident)
else:
return get_idents_defs(version, ident, family)
else:
return('Unknown subcommand: ' + cmd + '\n')
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and lib.isIdent(tok2)
and lib.compatibleFamily(
db.defs.get(tok2).get_families(), family)):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line, def_family = next(defs_this_ident)
ref_idx, ref_lines, ref_family = next(refs)
doc_idx, doc_line, doc_family = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line, def_family = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines, ref_family = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line, doc_family = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
if def_family == family or family == 'A':
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line, def_family = next(defs_this_ident)
if ref_idx == file_idx:
if lib.compatibleFamily(family, ref_family) or family == 'A':
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
