def _update_lang_with_ver(self, lang, ver=None, progress_cb=None):
"""Import stdlib data for this lang, if necessary.
"lang" is the language to update.
"ver" (optional) is a specific version of the language,
e.g. (5, 8).
"progress_cb" (optional) is a callable that is called as
follows to show the progress of the update:
progress_cb(<desc>, <value>)
where <desc> is a short string describing the current step
and <value> is an integer between 0 and 100 indicating the
level of completeness.
"""
log.debug("update '%s' stdlibs", lang)
# Figure out what updates need to be done...
if progress_cb:
try:
progress_cb("Determining necessary updates...", 5)
except:
log.exception("error in progress_cb (ignoring)")
if ver is not None:
ver_str = ".".join(map(str, ver))
cix_path = join(self.stdlibs_dir,
"%s-%s.cix" % (safe_lang_from_lang(lang), ver_str))
else:
cix_path = join(self.stdlibs_dir,
"%s.cix" % (safe_lang_from_lang(lang), ))
# Need to acquire db lock, as the indexer and main thread may both be
# calling into _update_lang_with_ver at the same time.
self.db.acquire_lock()
try:
todo = []
res = AreaResource(cix_path, "ci-pkg-dir")
try:
last_updated = self.res_index[res.area_path]
except KeyError:
todo.append(("add", res))
else:
mtime = os.stat(cix_path).st_mtime
if last_updated != mtime: # epsilon? '>=' instead of '!='?
todo.append(("update", res))
# ... and then do them.
self._handle_res_todos(lang, todo, progress_cb)
self.save()
finally:
self.db.release_lock()
def _update_lang_with_ver(self, lang, ver=None, progress_cb=None):
"""Import stdlib data for this lang, if necessary.
"lang" is the language to update.
"ver" (optional) is a specific version of the language,
e.g. (5, 8).
"progress_cb" (optional) is a callable that is called as
follows to show the progress of the update:
progress_cb(<desc>, <value>)
where <desc> is a short string describing the current step
and <value> is an integer between 0 and 100 indicating the
level of completeness.
"""
log.debug("update '%s' stdlibs", lang)
# Figure out what updates need to be done...
if progress_cb:
try:
progress_cb("Determining necessary updates...", 5)
except:
log.exception("error in progress_cb (ignoring)")
if ver is not None:
ver_str = ".".join(map(str, ver))
cix_path = join(self.stdlibs_dir,
"%s-%s.cix" % (safe_lang_from_lang(lang), ver_str))
else:
cix_path = join(self.stdlibs_dir,
"%s.cix" % (safe_lang_from_lang(lang), ))
# Need to acquire db lock, as the indexer and main thread may both be
# calling into _update_lang_with_ver at the same time.
self.db.acquire_lock()
try:
todo = []
res = AreaResource(cix_path, "ci-pkg-dir")
try:
last_updated = self.res_index[res.area_path]
except KeyError:
todo.append(("add", res))
else:
mtime = os.stat(cix_path).st_mtime
if last_updated != mtime: # epsilon? '>=' instead of '!='?
todo.append(("update", res))
# ... and then do them.
self._handle_res_todos(lang, todo, progress_cb)
self.save()
finally:
self.db.release_lock()
def _handle_res_todos(self, lang, todo, progress_cb=None):
if not todo:
return
for i, (action, res) in enumerate(todo):
cix_path = res.path
name = splitext(basename(cix_path))[0]
if '-' in name:
base, ver_str = name.split('-', 1)
ver = _ver_from_ver_str(ver_str)
else:
base = name
ver = None
assert base == safe_lang_from_lang(lang)
log.debug("%s %s stdlib: `%s'", action, name, cix_path)
verb = {"add": "Adding", "remove": "Removing",
"update": "Updating"}[action]
desc = "%s %s stdlib" % (verb, name)
if progress_cb:
try: progress_cb(desc, (5 + 95/len(todo)*i))
except: log.exception("error in progress_cb (ignoring)")
else:
self.db.report_event(desc)
if action == "add":
self._add_res(res, lang, name, ver)
elif action == "remove":
self._remove_res(res, lang, name, ver)
elif action == "update":
#XXX Bad for filesystem. Change this to do it
# more intelligently if possible.
self._remove_res(res, lang, name, ver)
self._add_res(res, lang, name, ver)
def lpaths_from_lang_and_blobname(self, lang, blobname):
"""Get lpaths for the named blob.
We get it from the blob's "lpaths" cache key (calculating that
if necessary).
"""
blob = self.get_blob(lang, blobname, look_in_cache_only=True)
if blob is not None:
if "lpaths" in blob.cache:
return blob.cache["lpaths"]
else:
blob = self.get_blob(lang, blobname)
if blob is None:
raise NotFoundInDatabase("%s '%s' blob not found in catalogs"
% (lang, blobname))
if "lpaths" in blob.cache:
return blob.cache["lpaths"]
# Need to calculate lpaths from 'blob'.
log.debug("calc symbol info for %s '%s' catalog blob", lang, blobname)
langintel = self.mgr.langintel_from_lang(lang)
lpaths = langintel.lpaths_from_blob(blob)
# Update cache and queue this up to be saved to disk (by .save()).
blob.cache["lpaths"] = lpaths
dbfile, res_id = self.blob_index[lang][blobname]
self._lock.acquire()
try:
self._dbsubpaths_and_lpaths_to_save.append(
(join(safe_lang_from_lang(lang), dbfile+".lpaths"), lpaths)
)
finally:
self._lock.release()
return lpaths
def lpaths_from_lang_and_blobname(self, lang, blobname):
"""Get lpaths for the named blob.
We get it from the blob's "lpaths" cache key (calculating that
if necessary).
"""
blob = self.get_blob(lang, blobname, look_in_cache_only=True)
if blob is not None:
if "lpaths" in blob.cache:
return blob.cache["lpaths"]
else:
blob = self.get_blob(lang, blobname)
if blob is None:
raise NotFoundInDatabase("%s '%s' blob not found in catalogs" %
(lang, blobname))
if "lpaths" in blob.cache:
return blob.cache["lpaths"]
# Need to calculate lpaths from 'blob'.
log.debug("calc symbol info for %s '%s' catalog blob", lang, blobname)
langintel = self.mgr.langintel_from_lang(lang)
lpaths = langintel.lpaths_from_blob(blob)
# Update cache and queue this up to be saved to disk (by .save()).
blob.cache["lpaths"] = lpaths
dbfile, res_id = self.blob_index[lang][blobname]
self._lock.acquire()
try:
self._dbsubpaths_and_lpaths_to_save.append(
(join(safe_lang_from_lang(lang), dbfile + ".lpaths"), lpaths))
finally:
self._lock.release()
return lpaths
def _handle_res_todos(self, lang, todo, progress_cb=None):
if not todo:
return
for i, (action, res) in enumerate(todo):
cix_path = res.path
name = splitext(basename(cix_path))[0]
if "-" in name:
base, ver_str = name.split("-", 1)
ver = _ver_from_ver_str(ver_str)
else:
base = name
ver = None
assert base == safe_lang_from_lang(lang)
log.debug("%s %s stdlib: `%s'", action, name, cix_path)
verb = {"add": "Adding", "remove": "Removing", "update": "Updating"}[action]
desc = "%s %s stdlib" % (verb, name)
if progress_cb:
try:
progress_cb(desc, (5 + 95 / len(todo) * i))
except:
log.exception("error in progress_cb (ignoring)")
else:
self.db.report_event(desc)
if action == "add":
self._add_res(res, lang, name, ver)
elif action == "remove":
self._remove_res(res, lang, name, ver)
elif action == "update":
# XXX Bad for filesystem. Change this to do it
# more intelligently if possible.
self._remove_res(res, lang, name, ver)
self._add_res(res, lang, name, ver)
def _remove_res(self, res):
LEN_PREFIX = self.db.LEN_PREFIX
res_id, last_updated, name, res_data = self.res_index[res.area_path]
# res_data: {lang -> blobname -> ilk -> toplevelnames}
for lang, tfifb in res_data.items():
dbfile_and_res_id_from_blobname = self.blob_index[lang]
for blobname, toplevelnames_from_ilk in tfifb.items():
# Update 'blob_index' for $lang.
dbfile, res_id = dbfile_and_res_id_from_blobname[blobname]
del dbfile_and_res_id_from_blobname[blobname]
# Remove ".blob" file (and associated caches).
pattern = join(self.base_dir, safe_lang_from_lang(lang),
dbfile+".*")
try:
for path in glob(pattern):
log.debug("fs-write: remove catalog %s blob file '%s'",
lang, basename(path))
os.remove(path)
except EnvironmentError as ex:
#XXX If get lots of these, then try harder. Perhaps
# creating a zombies area, or creating a list of
# them: self.db.add_zombie(dbpath).
#XXX THis isn't a correct analysis: the dbfile may just
# not have been there.
log.warn("could not remove dbfile '%s' (%s '%s'): "
"leaving zombie", dbpath, lang, blobname)
# Update 'toplevel*_index' for $lang.
# toplevelname_index: {lang -> ilk -> toplevelname -> res_id -> blobnames}
# toplevelprefix_index: {lang -> ilk -> prefix -> res_id -> toplevelnames}
for ilk, toplevelnames in six.iteritems(toplevelnames_from_ilk):
try:
bfrft = self.toplevelname_index[lang][ilk]
for toplevelname in toplevelnames:
del bfrft[toplevelname][res_id]
if not bfrft[toplevelname]:
del bfrft[toplevelname]
except KeyError as ex:
self.db.corruption("CatalogsZone._remove_res",
"error removing top-level names of ilk '%s' for "
"'%s' resource from toplevelname_index: %s"
% (ilk, basename(res.path), ex),
"ignore")
try:
tfrfp = self.toplevelprefix_index[lang][ilk]
for toplevelname in toplevelnames:
prefix = toplevelname[:LEN_PREFIX]
del tfrfp[prefix][res_id]
if not tfrfp[prefix]:
del tfrfp[prefix]
except KeyError as ex:
self.db.corruption("CatalogsZone._remove_res",
"error removing top-level name of ilk '%s' for "
"'%s' resource from toplevelprefix_index: %s"
% (ilk, basename(res.path), ex),
"ignore")
del self.res_index[res.area_path]
def _upgrade_wipe_db_langzones(self, curr_ver, result_ver):
for lang in self._gen_langs_in_db():
safe_lang = safe_lang_from_lang(lang)
langzone_dir = join(self.base_dir, "db", safe_lang)
if exists(langzone_dir):
log.debug("fs-write: wipe db/%s", safe_lang)
rmdir(langzone_dir)
open(join(self.base_dir, "VERSION"), 'w').write(result_ver)
def _upgrade_wipe_db_langzones(self, curr_ver, result_ver):
for lang in self._gen_langs_in_db():
safe_lang = safe_lang_from_lang(lang)
langzone_dir = join(self.base_dir, "db", safe_lang)
if exists(langzone_dir):
log.debug("fs-write: wipe db/%s", safe_lang)
rmdir(langzone_dir)
open(join(self.base_dir, "VERSION"), 'w').write(result_ver)
def vers_and_names_from_lang(self, lang):
"Returns an ordered list of (ver, name) for the given lang."
# _vers_and_names_from_lang = {
# "php": [
# ((4,3), "php-4.3"),
# ((5.0), "php-5.0"),
# ((5.1), "php-5.1"),
# ((5.2), "php-5.2"),
# ((5,3), "php-5.3")
# ],
# "ruby": [
# (None, "ruby"),
# ],
# ...
# }
vers_and_names = self._vers_and_names_from_lang.get(lang)
if vers_and_names is None:
# Find the available stdlibs for this language.
cix_glob = join(self.stdlibs_dir,
safe_lang_from_lang(lang) + "*.cix")
zip_glob = join(self.stdlibs_dir,
safe_lang_from_lang(lang) + "*.zip")
cix_paths = glob(cix_glob)
zip_paths = glob(zip_glob)
vers_and_names = []
for cix_path in (cix_paths + zip_paths):
name = splitext(basename(cix_path))[0]
if '-' in name:
base, ver_str = name.split('-', 1)
ver = _ver_from_ver_str(ver_str)
else:
base = name
ver = None
if base.lower() != lang.lower():
# Only process when the base name matches the language.
# I.e. skip if base is "python3" and lang is "python".
continue
vers_and_names.append((ver, name))
vers_and_names.sort()
self._vers_and_names_from_lang[lang] = vers_and_names
return vers_and_names
def _fillScanInputsTestCase():
for dpath, dnames, fnames in os.walk(gInputsDir):
# Don't descend into SCC control dirs.
scc_dirs = [".svn", "CVS", ".hg", ".git"]
for scc_dir in scc_dirs:
if scc_dir in dnames:
dnames.remove(scc_dir)
if dpath == gInputsDir and "unicode" in dnames:
# The scan_inputs/unicode is where the unicode test files
# are placed. Don't descend into here. They are handled elsewhere.
dnames.remove("unicode")
if ".svn" in dpath.split(os.sep):
# Skip subversion directories.
continue
for fname in fnames:
fpath = os.path.join(dpath, fname)[len(gInputsDir) + len(os.sep):]
if not isfile(join(dpath, fname)):
# With our Unicode testing we may have a directory that
# Python's os.walk() doesn't recognize as a dir, defaults to
# a file and hands it to us here. Skip those.
continue
if fname == ".DS_Store": continue
if fpath.endswith(".swp"): continue
if fpath.endswith("~"): continue
if fpath.endswith("__pycache__"): continue
if fpath.endswith(".pyc"): continue
if fpath.endswith(".pyo"): continue
if fpath.endswith(".pod"): continue
if fpath.endswith(".options"): continue # skip input option files
if fpath.endswith(".tags"): continue # skip tags files
lang = guess_lang_from_path(fpath)
# Manual hack to detect as Python 3.
if lang == "Python" and "py3" in fpath:
lang = "Python3"
safe_lang = safe_lang_from_lang(lang)
# Set tags for this test case.
tags = [safe_lang]
tagspath = join(dpath, fname + ".tags") # ws-separate set of tags
if exists(tagspath):
tags += open(tagspath, 'r').read().split()
def makeTestFunction(fpath_, tags_):
testFunction \
= lambda self, fpath=fpath_: _testOneInputFile(self, fpath_, tags=tags_)
testFunction.tags = tags_
return testFunction
name = "test_path:" + fpath
setattr(ScanInputsTestCase, name, makeTestFunction(fpath, tags))
_addUnicodeScanInputTests()
def _fillScanInputsTestCase():
for dpath, dnames, fnames in os.walk(gInputsDir):
# Don't descend into SCC control dirs.
scc_dirs = [".svn", "CVS", ".hg", ".git"]
for scc_dir in scc_dirs:
if scc_dir in dnames:
dnames.remove(scc_dir)
if dpath == gInputsDir and "unicode" in dnames:
# The scan_inputs/unicode is where the unicode test files
# are placed. Don't descend into here. They are handled elsewhere.
dnames.remove("unicode")
if ".svn" in dpath.split(os.sep):
# Skip subversion directories.
continue
for fname in fnames:
fpath = os.path.join(dpath, fname)[len(gInputsDir)+len(os.sep):]
if not isfile(join(dpath, fname)):
# With our Unicode testing we may have a directory that
# Python's os.walk() doesn't recognize as a dir, defaults to
# a file and hands it to us here. Skip those.
continue
if fname == ".DS_Store": continue
if fpath.endswith(".swp"): continue
if fpath.endswith("~"): continue
if fpath.endswith("__pycache__"): continue
if fpath.endswith(".pyc"): continue
if fpath.endswith(".pyo"): continue
if fpath.endswith(".pod"): continue
if fpath.endswith(".options"): continue # skip input option files
if fpath.endswith(".tags"): continue # skip tags files
lang = guess_lang_from_path(fpath)
# Manual hack to detect as Python 3.
if lang == "Python" and "py3" in fpath:
lang = "Python3"
safe_lang = safe_lang_from_lang(lang)
# Set tags for this test case.
tags = [safe_lang]
tagspath = join(dpath, fname + ".tags") # ws-separate set of tags
if exists(tagspath):
tags += open(tagspath, 'r').read().split()
def makeTestFunction(fpath_, tags_):
testFunction \
= lambda self, fpath=fpath_: _testOneInputFile(self, fpath_, tags=tags_)
testFunction.tags = tags_
return testFunction
name = "test_path:"+fpath
setattr(ScanInputsTestCase, name, makeTestFunction(fpath, tags))
_addUnicodeScanInputTests()
def vers_and_names_from_lang(self, lang):
"Returns an ordered list of (ver, name) for the given lang."
# _vers_and_names_from_lang = {
# "php": [
# ((4,3), "php-4.3"),
# ((5.0), "php-5.0"),
# ((5.1), "php-5.1"),
# ((5.2), "php-5.2"),
# ((5,3), "php-5.3")
# ],
# "ruby": [
# (None, "ruby"),
# ],
# ...
# }
vers_and_names = self._vers_and_names_from_lang.get(lang)
if vers_and_names is None:
# Find the available stdlibs for this language.
cix_glob = join(self.stdlibs_dir, safe_lang_from_lang(lang) + "*.cix")
zip_glob = join(self.stdlibs_dir, safe_lang_from_lang(lang) + "*.zip")
cix_paths = glob(cix_glob)
zip_paths = glob(zip_glob)
vers_and_names = []
for cix_path in cix_paths + zip_paths:
name = splitext(basename(cix_path))[0]
if "-" in name:
base, ver_str = name.split("-", 1)
ver = _ver_from_ver_str(ver_str)
else:
base = name
ver = None
if base.lower() != lang.lower():
# Only process when the base name matches the language.
# I.e. skip if base is "python3" and lang is "python".
continue
vers_and_names.append((ver, name))
vers_and_names.sort()
self._vers_and_names_from_lang[lang] = vers_and_names
return vers_and_names
def remove_lang(self, lang):
"""Remove the given language from the stdlib zone."""
log.debug("update '%s' stdlibs", lang)
# Figure out what updates need to be done...
cix_glob = join(self.stdlibs_dir, safe_lang_from_lang(lang) + "*.cix")
todo = []
for area, subpath in self.res_index:
res = AreaResource(subpath, area)
if fnmatch.fnmatch(res.path, cix_glob):
todo.append(("remove", AreaResource(subpath, area)))
# ... and then do them.
self._handle_res_todos(lang, todo)
self.save()
def remove_lang(self, lang):
"""Remove the given language from the stdlib zone."""
log.debug("update '%s' stdlibs", lang)
# Figure out what updates need to be done...
cix_glob = join(self.stdlibs_dir, safe_lang_from_lang(lang) + "*.cix")
todo = []
for area, subpath in self.res_index:
res = AreaResource(subpath, area)
if fnmatch.fnmatch(res.path, cix_glob):
todo.append(("remove", AreaResource(subpath, area)))
# ... and then do them.
self._handle_res_todos(lang, todo)
self.save()
def _upgrade_wipe_db_langs(self, curr_ver, result_ver, langs):
for lang in langs:
safe_lang = safe_lang_from_lang(lang)
# stdlibs zone
self.get_stdlibs_zone().remove_lang(lang)
# API catalogs zone
# TODO: CatalogsZone needs a .remove_lang(). Until then we just
# remove the whole thing.
# (multi)langzone
langzone_dir = join(self.base_dir, "db", safe_lang)
if exists(langzone_dir):
log.debug("fs-write: wipe db/%s", safe_lang)
rmdir(langzone_dir)
catalog_dir = join(self.base_dir, "db", "catalogs")
if exists(catalog_dir):
log.debug("fs-write: wipe db/catalogs")
rmdir(catalog_dir)
open(join(self.base_dir, "VERSION"), 'w').write(result_ver)
def _upgrade_wipe_db_langs(self, curr_ver, result_ver, langs):
for lang in langs:
safe_lang = safe_lang_from_lang(lang)
# stdlibs zone
self.get_stdlibs_zone().remove_lang(lang)
# API catalogs zone
# TODO: CatalogsZone needs a .remove_lang(). Until then we just
# remove the whole thing.
# (multi)langzone
langzone_dir = join(self.base_dir, "db", safe_lang)
if exists(langzone_dir):
log.debug("fs-write: wipe db/%s", safe_lang)
rmdir(langzone_dir)
catalog_dir = join(self.base_dir, "db", "catalogs")
if exists(catalog_dir):
log.debug("fs-write: wipe db/catalogs")
rmdir(catalog_dir)
open(join(self.base_dir, "VERSION"), 'w').write(result_ver)
def get_blob(self, lang, blobname, look_in_cache_only=False):
try:
dbfile, res_id = self.blob_index[lang][blobname]
except KeyError:
return None
# If index path is in the cache: return it, update its atime.
now = time.time()
blob_and_atime_from_blobname \
= self._blob_and_atime_from_blobname_from_lang_cache.setdefault(lang, {})
if blobname in blob_and_atime_from_blobname:
log.debug("cache-read: load %s blob `%s'", lang, blobname)
blob, atime = blob_and_atime_from_blobname[blobname]
blob_and_atime_from_blobname[blobname] = (blob, now)
return blob
# Need to load and cache it.
if look_in_cache_only:
return None
dbsubpath = join(self.base_dir, safe_lang_from_lang(lang), dbfile)
blob = self.db.load_blob(dbsubpath)
blob_and_atime_from_blobname[blobname] = (blob, now)
return blob
def get_blob(self, lang, blobname, look_in_cache_only=False):
try:
dbfile, res_id = self.blob_index[lang][blobname]
except KeyError:
return None
# If index path is in the cache: return it, update its atime.
now = time.time()
blob_and_atime_from_blobname \
= self._blob_and_atime_from_blobname_from_lang_cache.setdefault(lang, {})
if blobname in blob_and_atime_from_blobname:
log.debug("cache-read: load %s blob `%s'", lang, blobname)
blob, atime = blob_and_atime_from_blobname[blobname]
blob_and_atime_from_blobname[blobname] = (blob, now)
return blob
# Need to load and cache it.
if look_in_cache_only:
return None
dbsubpath = join(self.base_dir, safe_lang_from_lang(lang), dbfile)
blob = self.db.load_blob(dbsubpath)
blob_and_atime_from_blobname[blobname] = (blob, now)
return blob
def _remove_res(self, res):
LEN_PREFIX = self.db.LEN_PREFIX
res_id, last_updated, name, res_data = self.res_index[res.area_path]
# res_data: {lang -> blobname -> ilk -> toplevelnames}
for lang, tfifb in list(res_data.items()):
dbfile_and_res_id_from_blobname = self.blob_index[lang]
for blobname, toplevelnames_from_ilk in list(tfifb.items()):
# Update 'blob_index' for $lang.
dbfile, res_id = dbfile_and_res_id_from_blobname[blobname]
del dbfile_and_res_id_from_blobname[blobname]
# Remove ".blob" file (and associated caches).
pattern = join(self.base_dir, safe_lang_from_lang(lang),
dbfile + ".*")
try:
for path in glob(pattern):
log.debug("fs-write: remove catalog %s blob file '%s'",
lang, basename(path))
os.remove(path)
except EnvironmentError as ex:
# XXX If get lots of these, then try harder. Perhaps
# creating a zombies area, or creating a list of
# them: self.db.add_zombie(dbpath).
# XXX THis isn't a correct analysis: the dbfile may just
# not have been there.
log.warn(
"could not remove dbfile '%s' (%s '%s'): "
"leaving zombie", dbpath, lang, blobname)
# Update 'toplevel*_index' for $lang.
# toplevelname_index: {lang -> ilk -> toplevelname -> res_id -> blobnames}
# toplevelprefix_index: {lang -> ilk -> prefix -> res_id ->
# toplevelnames}
for ilk, toplevelnames in toplevelnames_from_ilk.items():
try:
bfrft = self.toplevelname_index[lang][ilk]
for toplevelname in toplevelnames:
del bfrft[toplevelname][res_id]
if not bfrft[toplevelname]:
del bfrft[toplevelname]
except KeyError as ex:
self.db.corruption(
"CatalogsZone._remove_res",
"error removing top-level names of ilk '%s' for "
"'%s' resource from toplevelname_index: %s" %
(ilk, basename(res.path), ex), "ignore")
try:
tfrfp = self.toplevelprefix_index[lang][ilk]
for toplevelname in toplevelnames:
prefix = toplevelname[:LEN_PREFIX]
del tfrfp[prefix][res_id]
if not tfrfp[prefix]:
del tfrfp[prefix]
except KeyError as ex:
self.db.corruption(
"CatalogsZone._remove_res",
"error removing top-level name of ilk '%s' for "
"'%s' resource from toplevelprefix_index: %s" %
(ilk, basename(res.path), ex), "ignore")
del self.res_index[res.area_path]
def _add_res(self, res):
cix_path = res.path
try:
tree = tree_from_cix_path(cix_path)
except ET.XMLParserError as ex:
log.warn("could not load `%s' into catalog (skipping): %s",
cix_path, ex)
return
LEN_PREFIX = self.db.LEN_PREFIX
res_id = self._new_res_id()
res_data = {} # {lang -> blobname -> ilk -> toplevelnames}
name = tree.get("name") or splitext(basename(cix_path))[0]
for blob in tree.findall("file/scope"):
lang, blobname = blob.get("lang"), blob.get("name")
if not lang:
raise DatabaseError("add `%s': no 'lang' attr on %r"
% (res, blob))
# Create 'res_data'.
tfifb = res_data.setdefault(lang, {})
toplevelnames_from_ilk = tfifb.setdefault(blobname, {})
if lang in self.db.import_everything_langs:
for toplevelname, elem in six.iteritems(blob.names):
ilk = elem.get("ilk") or elem.tag
if ilk not in toplevelnames_from_ilk:
toplevelnames_from_ilk[ilk] = set([toplevelname])
else:
toplevelnames_from_ilk[ilk].add(toplevelname)
# Update 'toplevel*_index'.
# toplevelname_index: {lang -> ilk -> toplevelname -> res_id -> blobnames}
# toplevelprefix_index: {lang -> ilk -> prefix -> res_id -> toplevelnames}
bfrftfi = self.toplevelname_index.setdefault(lang, {})
tfrfpfi = self.toplevelprefix_index.setdefault(lang, {})
for ilk, toplevelnames in six.iteritems(toplevelnames_from_ilk):
bfrft = bfrftfi.setdefault(ilk, {})
tfrfp = tfrfpfi.setdefault(ilk, {})
for toplevelname in toplevelnames:
bfr = bfrft.setdefault(toplevelname, {})
if res_id not in bfr:
bfr[res_id] = set([blobname])
else:
bfr[res_id].add(blobname)
prefix = toplevelname[:LEN_PREFIX]
tfr = tfrfp.setdefault(prefix, {})
if res_id not in tfr:
tfr[res_id] = set([toplevelname])
else:
tfr[res_id].add(toplevelname)
# Update 'blob_index'.
dbfile_and_res_id_from_blobname \
= self.blob_index.setdefault(lang, {})
assert blobname not in dbfile_and_res_id_from_blobname, \
("codeintel: %s %r blob in `%s' collides "
"with existing %s %r blob (from res_id %r) in catalog: "
"(XXX haven't decided how to deal with that yet)"
% (lang, blobname, cix_path, lang, blobname,
dbfile_and_res_id_from_blobname[blobname][1]))
dbfile = self.db.bhash_from_blob_info(cix_path, lang, blobname)
dbfile_and_res_id_from_blobname[blobname] = (dbfile, res_id)
# Write out '.blob' file.
dbdir = join(self.base_dir, safe_lang_from_lang(lang))
if not exists(dbdir):
log.debug("fs-write: mkdir '%s'", dbdir)
os.makedirs(dbdir)
log.debug("fs-write: catalog %s blob '%s'", lang, dbfile)
ET.ElementTree(blob).write(join(dbdir, dbfile+".blob"))
# Update 'res_index'.
last_updated = os.stat(cix_path).st_mtime
self.res_index[res.area_path] \
= (res_id, last_updated, name, res_data)
def _add_res(self, res):
cix_path = res.path
try:
tree = tree_from_cix_path(cix_path)
except ET.XMLParserError as ex:
log.warn("could not load `%s' into catalog (skipping): %s",
cix_path, ex)
return
LEN_PREFIX = self.db.LEN_PREFIX
res_id = self._new_res_id()
res_data = {} # {lang -> blobname -> ilk -> toplevelnames}
name = tree.get("name") or splitext(basename(cix_path))[0]
for blob in tree.findall("file/scope"):
lang, blobname = blob.get("lang"), blob.get("name")
if not lang:
raise DatabaseError("add `%s': no 'lang' attr on %r" %
(res, blob))
# Create 'res_data'.
tfifb = res_data.setdefault(lang, {})
toplevelnames_from_ilk = tfifb.setdefault(blobname, {})
if lang in self.db.import_everything_langs:
for toplevelname, elem in blob.names.items():
ilk = elem.get("ilk") or elem.tag
if ilk not in toplevelnames_from_ilk:
toplevelnames_from_ilk[ilk] = set([toplevelname])
else:
toplevelnames_from_ilk[ilk].add(toplevelname)
# Update 'toplevel*_index'.
# toplevelname_index: {lang -> ilk -> toplevelname -> res_id -> blobnames}
# toplevelprefix_index: {lang -> ilk -> prefix -> res_id ->
# toplevelnames}
bfrftfi = self.toplevelname_index.setdefault(lang, {})
tfrfpfi = self.toplevelprefix_index.setdefault(lang, {})
for ilk, toplevelnames in toplevelnames_from_ilk.items():
bfrft = bfrftfi.setdefault(ilk, {})
tfrfp = tfrfpfi.setdefault(ilk, {})
for toplevelname in toplevelnames:
bfr = bfrft.setdefault(toplevelname, {})
if res_id not in bfr:
bfr[res_id] = set([blobname])
else:
bfr[res_id].add(blobname)
prefix = toplevelname[:LEN_PREFIX]
tfr = tfrfp.setdefault(prefix, {})
if res_id not in tfr:
tfr[res_id] = set([toplevelname])
else:
tfr[res_id].add(toplevelname)
# Update 'blob_index'.
dbfile_and_res_id_from_blobname \
= self.blob_index.setdefault(lang, {})
assert blobname not in dbfile_and_res_id_from_blobname, \
("codeintel: %s %r blob in `%s' collides "
"with existing %s %r blob (from res_id %r) in catalog: "
"(XXX haven't decided how to deal with that yet)"
% (lang, blobname, cix_path, lang, blobname,
dbfile_and_res_id_from_blobname[blobname][1]))
dbfile = self.db.bhash_from_blob_info(cix_path, lang, blobname)
dbfile_and_res_id_from_blobname[blobname] = (dbfile, res_id)
# Write out '.blob' file.
dbdir = join(self.base_dir, safe_lang_from_lang(lang))
if not exists(dbdir):
log.debug("fs-write: mkdir '%s'", dbdir)
os.makedirs(dbdir)
log.debug("fs-write: catalog %s blob '%s'", lang, dbfile)
ET.ElementTree(blob).write(join(dbdir, dbfile + ".blob"))
# Update 'res_index'.
last_updated = os.stat(cix_path).st_mtime
self.res_index[res.area_path] \
= (res_id, last_updated, name, res_data)
