def worker(dirname, jobs, out_q):
result_dict = {}
files = {}
for job in jobs:
filename = job[0]
result_dict[filename] = apply_parse(job, basedir = dirname, **kwargs)
files[filename] = time.time()
parsed_dict = {}
for result in result_dict.values():
merge_recurse_inplace(parsed_dict, result)
parsed_dict[FILES] = files
out_q.put(parsed_dict)
def parse_all_single(self, sources, **kwargs):
pp = pprint.PrettyPrinter(indent=4)
parsed_dict = {}
parsed_dict[FILES] = {}
for job in sources:
filename = job[0]
result = apply_parse(job, basedir = self.basedir, **kwargs)
result[FILES] = {}
result[FILES][filename] = time.time()
merge_recurse_inplace(parsed_dict, result)
parsed_dict['basedir'] = self.basedir
return parsed_dict
def parse_all(self, sources, **kwargs):
# from http://eli.thegreenplace.net/2012/01/16/python-parallelizing-cpu-bound-tasks-with-multiprocessing
exclude_filters = '/usr/include'
sources = filter(lambda s: not s[0].startswith(exclude_filters), sources)
sources = filter(lambda s: not s[0].endswith('.c'), sources)
# sources = filter(lambda s: s[0].endswith('hello.cpp'), sources)
def worker(dirname, jobs, out_q):
result_dict = {}
files = {}
for job in jobs:
filename = job[0]
result_dict[filename] = apply_parse(job, basedir = dirname, **kwargs)
files[filename] = time.time()
parsed_dict = {}
for result in result_dict.values():
merge_recurse_inplace(parsed_dict, result)
parsed_dict[FILES] = files
out_q.put(parsed_dict)
out_q = mp.Queue()
nprocs = mp.cpu_count()
jobs = sources
chunksize = int(math.ceil(len(jobs) / float(nprocs)))
procs = []
for i in range(nprocs):
p = mp.Process(
target=worker,
args=(self.basedir, jobs[chunksize * i:chunksize * (i + 1)],
out_q))
procs.append(p)
p.start()
parsed_dict = {}
for i in range(nprocs):
result = out_q.get()
merge_recurse_inplace(parsed_dict, result)
for p in procs:
p.join()
parsed_dict['basedir'] = self.basedir
return parsed_dict
def parse_cursor(cursor, parsed_dict, ref_kind, basedir):
c = cursor
p = parsed_dict
filename = normalize_path(c.location.file.name)
assert(basedir)
basename = filename
if basedir:
idx = filename.find(basedir)
if idx != -1:
basename = filename[(idx + len(basedir)):]
locus = get_locus(c.location)
usr = c.get_usr()
filename_locus = get_filename_locus(basename, locus)
if c.kind in ref_kind:
assert not c.get_usr()
p.setdefault(basename, {})
assert(not basename.startswith(basedir))
file_dict = p[basename]
assert c.location.line and c.location.column
is_definition = c.is_definition()
if c.referenced and c.referenced.get_usr():
assert c.referenced.get_usr()
entry = {}
if c.kind == CursorKind.TEMPLATE_REF and \
c.referenced.kind == CursorKind.CLASS_TEMPLATE:
entry[TEMPLATE_USR] = c.referenced.get_usr()
elif is_ref_is_function_template(c, p, basedir):
entry[TEMPLATE_USR] = get_function_template_usr(c, p, basedir)
else:
entry[REF_USR] = c.referenced.get_usr()
if locus in file_dict:
merge_recurse_inplace(file_dict[locus], entry)
else:
file_dict[locus] = entry
# update reference in usr
r = c.referenced
r_usr = r.get_usr()
if TEMPLATE_USR in entry:
r_usr = entry[TEMPLATE_USR]
p.setdefault(usr, {})
filename_locus = get_filename_locus(basename, locus)
entry = {
REFS: [filename_locus]
}
if c.kind == CursorKind.CXX_BASE_SPECIFIER:
entry[KIND] = c.kind.name
entry[SPELL] = c.spelling
if r_usr in p:
merge_recurse_inplace(p[r_usr], entry)
else:
p[r_usr] = entry
if c.kind in function_kind:
# assert usr
entry = {
c.is_definition() and DEFI or DECL: filename_locus,
KIND: c.kind.name,
SPELL: c.spelling,
TYPE: c.type.kind.name,
REFS: [],
}
if usr in p:
merge_recurse_inplace(p[usr], entry)
else:
p[usr] = entry
p.setdefault(basename, {})
assert(not basename.startswith(basedir))
file_dict = p[basename]
locus_entry = {
USR: usr
}
if locus in file_dict:
merge_recurse_inplace(file_dict[locus], locus_entry)
else:
file_dict[locus] = locus_entry
result = handle_class_hierarchy(c, p)
sys.exit(0)
elif action == 'parse':
scanned_list = project.scan()
files = project.get_files_from_db(dbname)
pp.pprint(files)
if files:
scanned_list = project.scan_modified(scanned_list, files)
logging.debug('scanned_list: {}'.format([x[0] for x in scanned_list]))
parsed_dict = project.parse_all(scanned_list)
pp.pprint('Parsed {} in {} with keys'.format(len(parsed_dict), builddir))
pp.pprint(parsed_dict.keys())
elif action == 'parse_single':
scanned_list = project.scan()
files = project.get_files_from_db(dbname)
if files:
scanned_list = project.scan_modified(scanned_list, files)
parsed_dict = project.parse_all_single(scanned_list)
pp.pprint('Parsed {} in {} with keys'.format(len(parsed_dict), builddir))
pp.pprint(parsed_dict.keys())
storage = Storage(dbname, writeback=True)
storage_update = util.measure(storage.update)
merge_recurse_inplace(storage, parsed_dict, Storage)
storage.close()
