def builddb(treecfg, dbdir):
""" Post-process the build and make the SQL directory """
global big_blob
# We use this all over the place, cache it here.
plugins = dxr.get_active_plugins(treecfg)
# Building the database--this happens as multiple phases. In the first phase,
# we basically collect all of the information and organizes it. In the second
# phase, we link the data across multiple languages.
print "Post-processing the source files..."
big_blob = {}
srcdir = treecfg.sourcedir
objdir = treecfg.objdir
# srcdir=objdir is not a good idea
if os.path.normpath(srcdir) == objdir:
print('objdir must not be the same as srcdir')
sys.exit(1)
for plugin in plugins:
if 'post_process' in plugin.__all__:
big_blob[plugin.__name__] = plugin.post_process(srcdir, objdir)
# Save off the raw data blob
print "Storing data..."
dxr.store_big_blob(treecfg, big_blob)
# Build the sql for later queries. This is a combination of the main language
# schema as well as plugin-specific information. The pragmas that are
# executed should make the sql stage go faster.
print "Building SQL..."
dbname = dxr.get_database_filename(treecfg)
conn = dxr.open_database(dbname,
'PRAGMA synchronous=off; PRAGMA page_size=65536;')
# Import the schemata
schemata = [dxr.languages.get_standard_schema()]
for plugin in plugins:
schemata.append(plugin.get_schema())
conn.executescript('\n'.join(schemata))
conn.commit()
# Load and run the SQL
def sql_generator():
for statement in dxr.languages.get_sql_statements():
yield statement
for plugin in plugins:
if plugin.__name__ in big_blob:
plugblob = big_blob[plugin.__name__]
for statement in plugin.sqlify(plugblob):
yield statement
for stmt in sql_generator():
if isinstance(stmt, tuple):
conn.execute(stmt[0], stmt[1])
else:
conn.execute(stmt)
conn.commit()
conn.close()
def indextree(treecfg, doxref, dohtml, debugfile):
global big_blob
# If we're live, we'll need to move -current to -old; we'll move it back
# after we're done.
if treecfg.isdblive:
currentroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
oldroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
linkroot = os.path.join(treecfg.wwwdir, treecfg.tree)
if os.path.isdir(currentroot):
if os.path.exists(os.path.join(currentroot, '.dxr_xref', '.success')):
# Move current -> old, change link to old
try:
shutil.rmtree(oldroot)
except OSError:
assert_permission_denied()
pass
try:
shutil.move(currentroot, oldroot)
os.unlink(linkroot)
os.symlink(oldroot, linkroot)
except OSError:
assert_permission_denied()
pass
else:
# This current directory is bad, move it away
shutil.rmtree(currentroot)
# dxr xref files (index + sqlitedb) go in wwwdir/treename-current/.dxr_xref
# and we'll symlink it to wwwdir/treename later
htmlroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
dbdir = os.path.join(htmlroot, '.dxr_xref')
os.makedirs(dbdir, 0755)
dbname = dxr.get_database_filename(treecfg)
retcode = 0
if doxref:
builddb(treecfg, dbdir)
if treecfg.isdblive:
f = open(os.path.join(dbdir, '.success'), 'w')
f.close()
elif treecfg.isdblive:
# If the database is live, we need to copy database info from the old
# version of the code
oldhtml = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
olddbdir = os.path.join(oldhtml, '.dxr_xref')
shutil.rmtree(dbdir)
shutil.copytree(olddbdir, dbdir)
# Build static html
if dohtml:
if not doxref:
big_blob = dxr.load_big_blob(treecfg)
# Do we need to do file pivoting?
for plugin in dxr.get_active_plugins(treecfg):
if plugin.__name__ in big_blob:
plugin.pre_html_process(treecfg, big_blob[plugin.__name__])
dxr.htmlbuilders.build_htmlifier_map(dxr.get_active_plugins(treecfg))
treecfg.database = dbname
n = cpu_count()
p = Pool(processes=n)
print 'Building HTML files for %s...' % treecfg.tree, ; sys.stdout.flush ()
index_list = open(os.path.join(dbdir, "file_list.txt"), 'w')
file_list = []
def getOutputFiles():
for regular in treecfg.getFileList():
yield regular
filelist = set()
for plug in big_blob:
try:
filelist.update(big_blob[plug]["byfile"].keys())
except KeyError:
pass
for filename in filelist:
if filename.startswith("--GENERATED--/"):
relpath = filename[len("--GENERATED--/"):]
yield filename, os.path.join(treecfg.objdir, relpath)
if debugfile:
output_files = glob.glob (treecfg.sourcedir + '/' + debugfile)
if output_files == []:
print 'Error: Glob %s doesn\'t match any files' % debugfile
sys.exit (1)
last_dir = None
for f in getOutputFiles():
# In debug mode, we only care about some files
if debugfile and not treecfg.sourcedir + '/' + f[0] in output_files: continue
if os.path.dirname (treecfg.sourcedir + '/' + f[0]) != last_dir:
last_dir = os.path.dirname (treecfg.sourcedir + '/' + f[0])
print os.path.basename (last_dir), ; sys.stdout.flush ()
index_list.write(f[0] + '\n')
cpypath = os.path.join(htmlroot, f[0])
srcpath = f[1]
file_list.append(f)
# Make output directory
cpydir = os.path.dirname(cpypath)
if not os.path.exists(cpydir):
os.makedirs(cpydir)
def is_text(srcpath):
# xdg.Mime considers .lo as text, which is technically true but useless
if srcpath[-3:] == '.lo': return False
import xdg.Mime
mimetype = str(xdg.Mime.get_type (srcpath))
for valid in ['text', 'xml', 'shellscript', 'perl', 'm4', 'xbel']:
if valid in mimetype:
return True
return False
if not is_text(srcpath):
continue
p.apply_async(async_toHTML, [treecfg, srcpath, cpypath + ".html"])
if file_list == []:
print 'Error: No files found to index'
sys.exit (0)
print '...Done', '%d files indexed' % len (file_list)
p.apply_async(make_index, [file_list, dbdir])
index_list.close()
p.close()
p.join()
# Generate index.html files
# XXX: This wants to be parallelized. However, I seem to run into problems
# if it isn't.
def genhtml(treecfg, dirname, fnames):
make_index_html(treecfg, dirname, fnames, htmlroot)
os.path.walk(htmlroot, genhtml, treecfg)
# If the database is live, we need to switch the live to the new version
if treecfg.isdblive:
try:
try:
shutil.rmtree(oldroot)
except:
pass
try:
os.remove(linkroot + '.tmp')
except:
pass
os.symlink(currentroot, linkroot + '.tmp')
os.rename(linkroot + '.tmp', linkroot)
except OSError:
msg = sys.exc_info()[1] # Python 2/3 compatibility
print ('Replacing database failed: %s' % str(msg))
