def update_blob_ids(self, tag):
global hash_file_lock
if db.vars.exists('numBlobs'):
idx = db.vars.get('numBlobs')
else:
idx = 0
# Get blob hashes and associated file names (without path)
blobs = scriptLines('list-blobs', '-f', tag)
new_idxes = []
for blob in blobs:
hash, filename = blob.split(b' ', maxsplit=1)
if not db.blob.exists(hash):
db.blob.put(hash, idx)
with hash_file_lock:
db.hash.put(idx, hash)
db.file.put(idx, filename)
new_idxes.append(idx)
if verbose:
print(f"New blob #{idx} {hash}:{filename}")
idx += 1
db.vars.put('numBlobs', idx)
return new_idxes
def update_definitions(self, idxes):
global hash_file_lock, defs_lock, tags_defs
for idx in idxes:
if (idx % 1000 == 0): progress('defs: ' + str(idx), tags_defs[0])
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family in [None, 'M']: continue
lines = scriptLines('parse-defs', hash, filename, family)
with defs_lock:
for l in lines:
ident, type, line = l.split(b' ')
type = type.decode()
line = int(line.decode())
defs_idxes[idx*idx_key_mod + line] = ident
if db.defs.exists(ident):
obj = db.defs.get(ident)
elif lib.isIdent(ident):
obj = data.DefList()
else:
continue
obj.append(idx, type, line, family)
if verbose:
print(f"def {type} {ident} in #{idx} @ {line}")
db.defs.put(ident, obj)
def update_compatibles(self, idxes):
global hash_file_lock, comps_lock, tags_comps
for idx in idxes:
if (idx % 1000 == 0): progress('comps: ' + str(idx), tags_comps[0])
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family in [None, 'K', 'M']: continue
lines = compatibles_parser.run(scriptLines('get-blob', hash),
family)
comps = {}
for l in lines:
ident, line = l.split(' ')
if ident in comps:
comps[ident] += ',' + str(line)
else:
comps[ident] = str(line)
with comps_lock:
for ident, lines in comps.items():
if db.comps.exists(ident):
obj = db.comps.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines, family)
if verbose:
print(f"comps: {ident} in #{idx} @ {line}")
db.comps.put(ident, obj)
def updateReferences(idxes):
for idx in idxes:
if (idx % 1000 == 0): progress('refs: ' + str(idx))
hash = db.hash.get(idx)
filename = db.file.get(idx)
if not lib.hasSupportedExt(filename): continue
tokens = scriptLines('tokenize-file', '-b', hash)
even = True
lineNum = 1
idents = {}
for tok in tokens:
even = not even
if even:
if db.defs.exists(tok) and lib.isIdent(tok):
if tok in idents:
idents[tok] += ',' + str(lineNum)
else:
idents[tok] = str(lineNum)
else:
lineNum += tok.count(b'\1')
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines)
if verbose:
print(f"ref: {ident} in #{idx} @ {lines}")
db.refs.put(ident, obj)
def updateReferences(blobs):
for blob in blobs:
if (blob % 100 == 0): print('R:', blob)
hash = db.hash.get(blob)
filename = db.file.get(blob)
ext = filename[-2:]
if not (ext == '.c' or ext == '.h'): continue
tokens = scriptLines('tokenize-file', '-b', hash)
even = True
lineNum = 1
idents = {}
for tok in tokens:
even = not even
if even:
if db.defs.exists(tok) and lib.isIdent(tok):
if tok in idents:
idents[tok] += ',' + str(lineNum)
else:
idents[tok] = str(lineNum)
else:
lineNum += tok.count(b'\1')
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(blob, lines)
db.refs.put(ident, obj)
def update_definitions(self, idxes):
global hash_file_lock, defs_lock
for idx in idxes:
if (idx % 1000 == 0): progress('defs: ' + str(idx), self.index + 1)
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
lines = scriptLines('parse-defs', hash, filename, family)
for l in lines:
ident, type, line = l.split(b' ')
type = type.decode()
line = int(line.decode())
with defs_lock:
if db.defs.exists(ident):
obj = db.defs.get(ident)
else:
obj = data.DefList()
obj.add_family(family)
obj.append(idx, type, line, family)
if verbose:
print(f"def {type} {ident} in #{idx} @ {line}")
with defs_lock:
db.defs.put(ident, obj)
def update_doc_comments(self, idxes):
global hash_file_lock
for idx in idxes:
if (idx % 1000 == 0): progress('docs: ' + str(idx), self.index + 1)
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
lines = scriptLines('parse-docs', hash, filename)
for l in lines:
ident, line = l.split(b' ')
line = int(line.decode())
if db.docs.exists(ident):
obj = db.docs.get(ident)
else:
obj = data.RefList()
obj.append(idx, str(line), family)
if verbose:
print(f"doc: {ident} in #{idx} @ {line}")
db.docs.put(ident, obj)
def update_versions(self, tag):
global blobs_lock
# Get blob hashes and associated file paths
blobs = scriptLines('list-blobs', '-p', tag)
buf = []
for blob in blobs:
hash, path = blob.split(b' ', maxsplit=1)
with blobs_lock:
idx = db.blob.get(hash)
buf.append((idx, path))
buf = sorted(buf)
obj = PathList()
for idx, path in buf:
obj.append(idx, path)
# Store DT bindings documentation files to parse them later
if path[:33] == b'Documentation/devicetree/bindings':
bindings_idxes.append(idx)
if verbose:
print(f"Tag {tag}: adding #{idx} {path}")
db.vers.put(tag, obj, sync=True)
def update_references(self, idxes):
global hash_file_lock, defs_lock, refs_lock, tags_refs
for idx in idxes:
if (idx % 1000 == 0): progress('refs: ' + str(idx), tags_refs[0])
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
prefix = b''
# Kconfig values are saved as CONFIG_<value>
if family == 'K':
prefix = b'CONFIG_'
tokens = scriptLines('tokenize-file', '-b', hash, family)
even = True
line_num = 1
idents = {}
with defs_lock:
for tok in tokens:
even = not even
if even:
tok = prefix + tok
if (db.defs.exists(tok) and
not ( (idx*idx_key_mod + line_num) in defs_idxes and
defs_idxes[idx*idx_key_mod + line_num] == tok ) and
(family != 'M' or tok.startswith(b'CONFIG_'))):
# We only index CONFIG_??? in makefiles
if tok in idents:
idents[tok] += ',' + str(line_num)
else:
idents[tok] = str(line_num)
else:
line_num += tok.count(b'\1')
with refs_lock:
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines, family)
if verbose:
print(f"ref: {ident} in #{idx} @ {lines}")
db.refs.put(ident, obj)
def updateVersions(tag):
blobs = scriptLines('list-blobs', '-p', tag)
buf = []
for blob in blobs:
hash, path = blob.split(b' ', maxsplit=1)
idx = db.blob.get(hash)
buf.append((idx, path))
buf = sorted(buf)
obj = PathList()
for idx, path in buf:
obj.append(idx, path)
db.vers.put(tag, obj, sync=True)
def update_references(self, idxes):
global hash_file_lock, defs_lock
for idx in idxes:
if (idx % 1000 == 0): progress('refs: ' + str(idx), self.index + 1)
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = lib.getFileFamily(filename)
if family == None: continue
prefix = b''
# Kconfig values are saved as CONFIG_<value>
if family == 'K':
prefix = b'CONFIG_'
tokens = scriptLines('tokenize-file', '-b', hash, family)
even = True
line_num = 1
idents = {}
for tok in tokens:
even = not even
if even:
tok = prefix + tok
with defs_lock:
if db.defs.exists(tok) and lib.isIdent(tok):
if tok in idents:
idents[tok] += ',' + str(line_num)
else:
idents[tok] = str(line_num)
else:
line_num += tok.count(b'\1')
for ident, lines in idents.items():
if db.refs.exists(ident):
obj = db.refs.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines, family)
if verbose:
print(f"ref: {ident} in #{idx} @ {lines}")
db.refs.put(ident, obj)
def update_versions(self, tag):
# Get blob hashes and associated file paths
blobs = scriptLines('list-blobs', '-p', tag)
buf = []
for blob in blobs:
hash, path = blob.split(b' ', maxsplit=1)
idx = db.blob.get(hash)
buf.append((idx, path))
buf = sorted(buf)
obj = PathList()
for idx, path in buf:
obj.append(idx, path)
if verbose:
print(f"Tag {tag}: adding #{idx} {path}")
db.vers.put(tag, obj, sync=True)
def updateBlobIDs(tag):
if db.vars.exists('numBlobs'):
idx = db.vars.get('numBlobs')
else:
idx = 0
blobs = scriptLines('list-blobs', '-f', tag)
newBlobs = []
for blob in blobs:
hash, filename = blob.split(b' ', maxsplit=1)
if not db.blob.exists(hash):
db.blob.put(hash, idx)
db.hash.put(idx, hash)
db.file.put(idx, filename)
newBlobs.append(idx)
idx += 1
db.vars.put('numBlobs', idx)
return newBlobs
def updateDefinitions(blobs):
for blob in blobs:
if (blob % 1000 == 0): progress('defs: ' + str(blob))
hash = db.hash.get(blob)
filename = db.file.get(blob)
if not lib.hasSupportedExt(filename): continue
lines = scriptLines('parse-defs', hash, filename)
for l in lines:
ident, type, line = l.split(b' ')
type = type.decode()
line = int(line.decode())
if db.defs.exists(ident):
obj = db.defs.get(ident)
else:
obj = data.DefList()
obj.append(blob, type, line)
db.defs.put(ident, obj)
def updateDefinitions(blobs):
for blob in blobs:
if (blob % 100 == 0): print('D:', blob)
hash = db.hash.get(blob)
filename = db.file.get(blob)
ext = filename[-2:]
if not (ext == '.c' or ext == '.h'): continue
lines = scriptLines('parse-defs', hash, filename)
for l in lines:
ident, type, line = l.split(b' ')
type = type.decode()
line = int(line.decode())
if db.defs.exists(ident):
obj = db.defs.get(ident)
else:
obj = data.DefList()
obj.append(blob, type, line)
db.defs.put(ident, obj)
def updateDocComments(idxes):
for idx in idxes:
if (idx % 1000 == 0): progress('docs: ' + str(idx))
hash = db.hash.get(idx)
filename = db.file.get(idx)
if not lib.hasSupportedExt(filename): continue
lines = scriptLines('parse-docs', hash, filename)
for l in lines:
ident, line = l.split(b' ')
line = int(line.decode())
if db.docs.exists(ident):
obj = db.docs.get(ident)
else:
obj = data.RefList()
obj.append(idx, str(line))
if verbose:
print(f"doc: {ident} in #{idx} @ {line}")
db.docs.put(ident, obj)
def update_compatibles_bindings(self, idxes):
global hash_file_lock, comps_lock, comps_docs_lock, tags_comps_docs, bindings_idxes
for idx in idxes:
if (idx % 1000 == 0):
progress('comps_docs: ' + str(idx), tags_comps_docs[0])
if not idx in bindings_idxes: # Parse only bindings doc files
continue
with hash_file_lock:
hash = db.hash.get(idx)
filename = db.file.get(idx)
family = 'B'
lines = compatibles_parser.run(scriptLines('get-blob', hash),
family)
comps_docs = {}
with comps_lock:
for l in lines:
ident, line = l.split(' ')
if db.comps.exists(ident):
if ident in comps_docs:
comps_docs[ident] += ',' + str(line)
else:
comps_docs[ident] = str(line)
with comps_docs_lock:
for ident, lines in comps_docs.items():
if db.comps_docs.exists(ident):
obj = db.comps_docs.get(ident)
else:
obj = data.RefList()
obj.append(idx, lines, family)
if verbose:
print(f"comps_docs: {ident} in #{idx} @ {line}")
db.comps_docs.put(ident, obj)
def updateDefinitions(idxes):
for idx in idxes:
if (idx % 1000 == 0): progress('defs: ' + str(idx))
hash = db.hash.get(idx)
filename = db.file.get(idx)
if not lib.hasSupportedExt(filename): continue
lines = scriptLines('parse-defs', hash, filename)
for l in lines:
ident, type, line = l.split(b' ')
type = type.decode()
line = int(line.decode())
if db.defs.exists(ident):
obj = db.defs.get(ident)
else:
obj = data.DefList()
obj.append(idx, type, line)
if verbose:
print(f"def {type} {ident} in #{idx} @ {line}")
db.defs.put(ident, obj)
import lib
import data
import os
try:
dbDir = os.environ['LXR_DATA_DIR']
except KeyError:
print(argv[0] + ': LXR_DATA_DIR needs to be set')
exit(1)
db = data.DB(dbDir, readonly=True)
cmd = argv[1]
if cmd == 'versions':
for p in scriptLines('list-tags', '-r'):
if db.vers.exists(p):
echo(p + b'\n')
elif cmd == 'type':
version = argv[2]
path = argv[3]
p = script('get-type', version, path)
echo(p)
elif cmd == 'dir':
version = argv[2]
path = argv[3]
p = script('get-dir', version, path)
echo(p)
obj = data.RefList()
obj.append(idx, str(line), family)
if verbose:
print(f"doc: {ident} in #{idx} @ {line}")
db.docs.put(ident, obj)
def progress(msg, current):
print('{} - {} ({:.0%})'.format(project, msg, current / num_tags))
# Main
tag_buf = []
for tag in scriptLines('list-tags'):
if not db.vers.exists(tag):
tag_buf.append(tag)
num_tags = len(tag_buf)
project = lib.currentProject()
print(project + ' - found ' + str(len(tag_buf)) + ' new tags')
id_version_thread = UpdateIdVersion(tag_buf)
defs_thread = UpdateDefs()
refs_thread = UpdateRefs()
docs_thread = UpdateDocs()
#Start to process tags
id_version_thread.start()
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and lib.isIdent(tok2)
and lib.compatibleFamily(
db.defs.get(tok2).get_families(), family)):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line, def_family = next(defs_this_ident)
ref_idx, ref_lines, ref_family = next(refs)
doc_idx, doc_line, doc_family = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line, def_family = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines, ref_family = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line, doc_family = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
if def_family == family or family == 'A':
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line, def_family = next(defs_this_ident)
if ref_idx == file_idx:
if lib.compatibleFamily(family, ref_family) or family == 'A':
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
# TODO: this latest tag may have just been retrieved
# in the git repository and may not have been indexed yet
# This could results in failed queries
return decode(script('get-latest')).rstrip('\n')
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
if lib.hasSupportedExt(path):
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
symbol_definitions = []
symbol_references = []
symbol_doccomments = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references, symbol_doccomments
if not db.vers.exists(version):
return symbol_definitions, symbol_references, symbol_doccomments
files_this_version = db.vers.get(version).iter()
defs_this_ident = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs_this_ident and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
if db.docs.exists(ident):
docs = db.docs.get(ident).iter(dummy=True)
else:
docs = data.RefList().iter(dummy=True)
# vers, defs, refs, and docs are all populated by update.py in order of
# idx, and there is a one-to-one mapping between blob hashes and idx
# values. Therefore, we can sequentially step through the defs, refs,
# and docs for each file in a version.
def_idx, def_type, def_line = next(defs_this_ident)
ref_idx, ref_lines = next(refs)
doc_idx, doc_line = next(docs)
dBuf = []
rBuf = []
docBuf = []
for file_idx, file_path in files_this_version:
# Advance defs, refs, and docs to the current file
while def_idx < file_idx:
def_idx, def_type, def_line = next(defs_this_ident)
while ref_idx < file_idx:
ref_idx, ref_lines = next(refs)
while doc_idx < file_idx:
doc_idx, doc_line = next(docs)
# Copy information about this identifier into dBuf, rBuf, and docBuf.
while def_idx == file_idx:
dBuf.append((file_path, def_type, def_line))
def_idx, def_type, def_line = next(defs_this_ident)
if ref_idx == file_idx:
rBuf.append((file_path, ref_lines))
if doc_idx == file_idx: # TODO should this be a `while`?
docBuf.append((file_path, doc_line))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
for path, docline in sorted(docBuf):
symbol_doccomments.append(SymbolInstance(path, docline))
return symbol_definitions, symbol_references, symbol_doccomments
else:
return ('Unknown subcommand: ' + cmd + '\n')
def query (cmd, *args):
buffer = BytesIO()
def echo (arg):
buffer.write (arg)
if cmd == 'versions':
for p in scriptLines ('list-tags', '-h'):
if db.vers.exists (p.split(b' ')[2]):
echo (p + b'\n')
elif cmd == 'latest':
p = script ('get-latest')
echo (p)
elif cmd == 'type':
version = args[0]
path = args[1]
p = script ('get-type', version, path)
echo (p)
elif cmd == 'dir':
version = args[0]
path = args[1]
p = script ('get-dir', version, path)
echo (p)
elif cmd == 'file':
version = args[0]
path = args[1]
ext = os.path.splitext(path)[1]
if ext in ['.c', '.cc', '.cpp', '.h']:
tokens = scriptLines ('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists (tok) and lib.isIdent (tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape (tok)
echo (tok)
else:
p = script ('get-file', version, path)
echo (p)
elif cmd == 'ident':
version = args[0]
ident = args[1]
if not db.defs.exists (ident):
echo (('Unknown identifier: ' + ident + '\n').encode())
return buffer.getvalue()
if not db.vers.exists (version):
echo (('Unknown version: ' + version + '\n').encode())
return buffer.getvalue()
vers = db.vers.get (version).iter()
defs = db.defs.get (ident).iter (dummy=True)
# FIXME: see why we can have a discrepancy between defs and refs
if db.refs.exists (ident):
refs = db.refs.get (ident).iter (dummy=True)
else:
refs = data.RefList().iter (dummy=True)
id2, type, dline = next (defs)
id3, rlines = next (refs)
dBuf = []
rBuf = []
for id1, path in vers:
while id1 > id2:
id2, type, dline = next (defs)
while id1 > id3:
id3, rlines = next (refs)
while id1 == id2:
dBuf.append ((path, type, dline))
id2, type, dline = next (defs)
if id1 == id3:
rBuf.append ((path, rlines))
echo (('Defined in ' + str(len(dBuf)) + ' files:\n').encode())
for path, type, dline in sorted (dBuf):
echo ((path + ': ' + str (dline) + ' (' + type + ')\n').encode())
echo (('\nReferenced in ' + str(len(rBuf)) + ' files:\n').encode())
for path, rlines in sorted (rBuf):
echo ((path + ': ' + rlines + '\n').encode())
else:
echo (('Unknown subcommand: ' + cmd + '\n').encode())
return buffer.getvalue()
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu,tag = taginfo
elif (num ==3):
topmenu,submenu,tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
previous = None
tag = ''
index = 0
# If we get the same tag twice, we are at the oldest one
while not db.vers.exists(tag) and previous != tag:
previous = tag
tag = decode(script('get-latest', str(index))).rstrip('\n')
index += 1
return tag
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'exist':
# Returns True if the requested file exists, overwise returns False
version = args[0]
path = args[1]
dirname, filename = os.path.split(path)
entries = decode(script('get-dir', version, dirname)).split("\n")[:-1]
for entry in entries:
fname = entry.split(" ")[1]
if fname == filename:
return True
return False
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
filename = os.path.basename(path)
family = lib.getFileFamily(filename)
if family != None:
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path, family)
even = True
prefix = b''
if family == 'K':
prefix = b'CONFIG_'
for tok in tokens:
even = not even
tok2 = prefix + tok
if (even and db.defs.exists(tok2) and
(lib.compatibleFamily(db.defs.get(tok2).get_families(), family) or
lib.compatibleMacro(db.defs.get(tok2).get_macros(), family))):
tok = b'\033[31m' + tok2 + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'family':
# Get the family of a given file
filename = args[0]
return lib.getFileFamily(filename)
elif cmd == 'dts-comp':
# Get state of dts_comp_support
return dts_comp_support
elif cmd == 'dts-comp-exists':
# Check if a dts compatible string exists
ident = args[0]
if dts_comp_support:
return db.comps.exists(ident)
else:
return False
elif cmd == 'keys':
# Return all keys of a given database
# /!\ This can take a while /!\
name = args[0]
if name == 'vars':
return db.vars.get_keys()
elif name == 'blob':
return db.blob.get_keys()
elif name == 'hash':
return db.hash.get_keys()
elif name == 'file':
return db.file.get_keys()
elif name == 'vers':
return db.vers.get_keys()
elif name == 'defs':
return db.defs.get_keys()
elif name == 'refs':
return db.refs.get_keys()
elif name == 'docs':
return db.docs.get_keys()
elif name == 'comps' and dts_comp_support:
return db.comps.get_keys()
elif name == 'comps_docs' and dts_comp_support:
return db.comps_docs.get_keys()
else:
return []
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
family = args[2]
# DT bindings compatible strings are handled differently
if family == 'B':
return get_idents_comps(version, ident)
else:
return get_idents_defs(version, ident, family)
else:
return('Unknown subcommand: ' + cmd + '\n')
def query(cmd, *args):
if cmd == 'versions':
# Returns the list of indexed versions in the following format:
# topmenu submenu tag
# Example: v3 v3.1 v3.1-rc10
versions = OrderedDict()
for line in scriptLines('list-tags', '-h'):
taginfo = decode(line).split(' ')
num = len(taginfo)
topmenu, submenu = 'FIXME', 'FIXME'
if (num == 1):
tag, = taginfo
elif (num == 2):
submenu, tag = taginfo
elif (num == 3):
topmenu, submenu, tag = taginfo
if db.vers.exists(tag):
if topmenu not in versions:
versions[topmenu] = OrderedDict()
if submenu not in versions[topmenu]:
versions[topmenu][submenu] = []
versions[topmenu][submenu].append(tag)
return versions
elif cmd == 'latest':
# Returns the tag considered as the latest one
# TODO: this latest tag may have just been retrieved
# in the git repository and may not have been indexed yet
# This could results in failed queries
return decode(script('get-latest')).rstrip('\n')
elif cmd == 'type':
# Returns the type (blob or tree) associated to
# the given path. Example:
# > ./query.py type v3.1-rc10 /Makefile
# blob
# > ./query.py type v3.1-rc10 /arch
# tree
version = args[0]
path = args[1]
return decode(script('get-type', version, path)).strip()
elif cmd == 'dir':
# Returns the contents (trees or blobs) of the specified directory
# Example: ./query.py dir v3.1-rc10 /arch
version = args[0]
path = args[1]
entries_str = decode(script('get-dir', version, path))
return entries_str.split("\n")[:-1]
elif cmd == 'file':
# Returns the contents of the specified file
# Tokens are marked for further processing
# Example: ./query.py file v3.1-rc10 /Makefile
version = args[0]
path = args[1]
if lib.hasSupportedExt(path):
buffer = BytesIO()
tokens = scriptLines('tokenize-file', version, path)
even = True
for tok in tokens:
even = not even
if even and db.defs.exists(tok) and lib.isIdent(tok):
tok = b'\033[31m' + tok + b'\033[0m'
else:
tok = lib.unescape(tok)
buffer.write(tok)
return decode(buffer.getvalue())
else:
return decode(script('get-file', version, path))
elif cmd == 'ident':
# Returns identifier search results
version = args[0]
ident = args[1]
symbol_definitions = []
symbol_references = []
if not db.defs.exists(ident):
return symbol_definitions, symbol_references
if not db.vers.exists(version):
return symbol_definitions, symbol_references
vers = db.vers.get(version).iter()
defs = db.defs.get(ident).iter(dummy=True)
# FIXME: see why we can have a discrepancy between defs and refs
if db.refs.exists(ident):
refs = db.refs.get(ident).iter(dummy=True)
else:
refs = data.RefList().iter(dummy=True)
id2, type, dline = next(defs)
id3, rlines = next(refs)
dBuf = []
rBuf = []
for id1, path in vers:
while id1 > id2:
id2, type, dline = next(defs)
while id1 > id3:
id3, rlines = next(refs)
while id1 == id2:
dBuf.append((path, type, dline))
id2, type, dline = next(defs)
if id1 == id3:
rBuf.append((path, rlines))
for path, type, dline in sorted(dBuf):
symbol_definitions.append(SymbolInstance(path, dline, type))
for path, rlines in sorted(rBuf):
symbol_references.append(SymbolInstance(path, rlines))
return symbol_definitions, symbol_references
else:
return ('Unknown subcommand: ' + cmd + '\n')
