def builddb(treecfg, dbdir, tmproot):
""" Post-process the build and make the SQL directory """
global big_blob
# Build the sql for later queries. This is a combination of the main language
# schema as well as plugin-specific information. The pragmas that are
# executed should make the sql stage go faster.
print "Building SQL..."
conn = getdbconn(treecfg, dbdir)
# We use this all over the place, cache it here.
plugins = dxr.get_active_plugins(treecfg)
# Import the schemata
schemata = [dxr.languages.get_standard_schema()]
for plugin in plugins:
schemata.append(plugin.get_schema())
conn.executescript('\n'.join(schemata))
conn.commit()
# Building the database--this happens as multiple phases. In the first phase,
# we basically collect all of the information and organizes it. In the second
# phase, we link the data across multiple languages.
big_blob = {}
srcdir = treecfg.sourcedir
objdir = treecfg.objdir
for plugin in plugins:
cache = None
if 'post_process' in plugin.__all__:
big_blob[plugin.__name__] = cache = plugin.post_process(
srcdir, objdir)
if 'build_database' in plugin.__all__:
plugin.build_database(conn, srcdir, objdir, cache)
# Save off the raw data blob
# print "Storing data..."
# dxr.store_big_blob(treecfg, big_blob, tmproot)
# Load and run the SQL
# def sql_generator():
# for statement in dxr.languages.get_sql_statements():
# yield statement
# for plugin in plugins:
# if plugin.__name__ in big_blob:
# plugblob = big_blob[plugin.__name__]
# for statement in plugin.sqlify(plugblob):
# yield statement
#
# for stmt in sql_generator():
# if isinstance(stmt, tuple):
# conn.execute(stmt[0], stmt[1])
# else:
# conn.execute(stmt)
conn.commit()
conn.close()
def builddb(treecfg, dbdir):
""" Post-process the build and make the SQL directory """
global big_blob
# We use this all over the place, cache it here.
plugins = dxr.get_active_plugins(treecfg)
# Building the database--this happens as multiple phases. In the first phase,
# we basically collect all of the information and organizes it. In the second
# phase, we link the data across multiple languages.
print "Post-processing the source files..."
big_blob = {}
srcdir = treecfg.sourcedir
objdir = treecfg.objdir
# srcdir=objdir is not a good idea
if os.path.normpath(srcdir) == objdir:
print('objdir must not be the same as srcdir')
sys.exit(1)
for plugin in plugins:
if 'post_process' in plugin.__all__:
big_blob[plugin.__name__] = plugin.post_process(srcdir, objdir)
# Save off the raw data blob
print "Storing data..."
dxr.store_big_blob(treecfg, big_blob)
# Build the sql for later queries. This is a combination of the main language
# schema as well as plugin-specific information. The pragmas that are
# executed should make the sql stage go faster.
print "Building SQL..."
dbname = dxr.get_database_filename(treecfg)
conn = dxr.open_database(dbname,
'PRAGMA synchronous=off; PRAGMA page_size=65536;')
# Import the schemata
schemata = [dxr.languages.get_standard_schema()]
for plugin in plugins:
schemata.append(plugin.get_schema())
conn.executescript('\n'.join(schemata))
conn.commit()
# Load and run the SQL
def sql_generator():
for statement in dxr.languages.get_sql_statements():
yield statement
for plugin in plugins:
if plugin.__name__ in big_blob:
plugblob = big_blob[plugin.__name__]
for statement in plugin.sqlify(plugblob):
yield statement
for stmt in sql_generator():
if isinstance(stmt, tuple):
conn.execute(stmt[0], stmt[1])
else:
conn.execute(stmt)
conn.commit()
conn.close()
def builddb(treecfg, dbdir):
""" Post-process the build and make the SQL directory """
global big_blob
# We use this all over the place, cache it here.
plugins = dxr.get_active_plugins(treecfg)
# Building the database--this happens as multiple phases. In the first phase,
# we basically collect all of the information and organizes it. In the second
# phase, we link the data across multiple languages.
print "Post-processing the source files..."
big_blob = {}
srcdir = treecfg.sourcedir
objdir = treecfg.objdir
for plugin in plugins:
if 'post_process' in plugin.__all__:
big_blob[plugin.__name__] = plugin.post_process(srcdir, objdir)
# Save off the raw data blob
print "Storing data..."
dxr.store_big_blob(treecfg, big_blob)
# Build the sql for later queries. This is a combination of the main language
# schema as well as plugin-specific information. The pragmas that are
# executed should make the sql stage go faster.
print "Building SQL..."
dbname = treecfg.tree + '.sqlite'
conn = sqlite3.connect(os.path.join(dbdir, dbname))
conn.execute('PRAGMA synchronous=off')
conn.execute('PRAGMA page_size=65536')
# Safeguard against non-ASCII text. Let's just hope everyone uses UTF-8
conn.text_factory = str
# Import the schemata
schemata = [dxr.languages.get_standard_schema()]
for plugin in plugins:
schemata.append(plugin.get_schema())
conn.executescript('\n'.join(schemata))
conn.commit()
# Load and run the SQL
def sql_generator():
for statement in dxr.languages.get_sql_statements():
yield statement
for plugin in plugins:
if plugin.__name__ in big_blob:
plugblob = big_blob[plugin.__name__]
for statement in plugin.sqlify(plugblob):
yield statement
for stmt in sql_generator():
if isinstance(stmt, tuple):
print "%s:%s" % (stmt[0],stmt[1])
conn.execute(stmt[0], stmt[1])
else:
conn.execute(stmt)
conn.commit()
conn.close()
def builddb(treecfg, dbdir, tmproot):
""" Post-process the build and make the SQL directory """
global big_blob
# Build the sql for later queries. This is a combination of the main language
# schema as well as plugin-specific information. The pragmas that are
# executed should make the sql stage go faster.
print "Building SQL..."
conn = getdbconn(treecfg, dbdir)
# We use this all over the place, cache it here.
plugins = dxr.get_active_plugins(treecfg)
# Import the schemata
schemata = [dxr.languages.get_standard_schema()]
for plugin in plugins:
schemata.append(plugin.get_schema())
conn.executescript('\n'.join(schemata))
conn.commit()
# Building the database--this happens as multiple phases. In the first phase,
# we basically collect all of the information and organizes it. In the second
# phase, we link the data across multiple languages.
big_blob = {}
srcdir = treecfg.sourcedir
objdir = treecfg.objdir
for plugin in plugins:
cache = None
if 'post_process' in plugin.__all__:
big_blob[plugin.__name__] = cache = plugin.post_process(srcdir, objdir)
if 'build_database' in plugin.__all__:
plugin.build_database(conn, srcdir, objdir, cache)
# Save off the raw data blob
# print "Storing data..."
# dxr.store_big_blob(treecfg, big_blob, tmproot)
# Load and run the SQL
# def sql_generator():
# for statement in dxr.languages.get_sql_statements():
# yield statement
# for plugin in plugins:
# if plugin.__name__ in big_blob:
# plugblob = big_blob[plugin.__name__]
# for statement in plugin.sqlify(plugblob):
# yield statement
#
# for stmt in sql_generator():
# if isinstance(stmt, tuple):
# conn.execute(stmt[0], stmt[1])
# else:
# conn.execute(stmt)
conn.commit()
conn.close()
def indextree(treecfg, doxref, dohtml, debugfile):
global big_blob
# If we're live, we'll need to move -current to -old; we'll move it back
# after we're done.
if treecfg.isdblive:
currentroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
oldroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
linkroot = os.path.join(treecfg.wwwdir, treecfg.tree)
if os.path.isdir(currentroot):
if os.path.exists(os.path.join(currentroot, '.dxr_xref', '.success')):
# Move current -> old, change link to old
try:
shutil.rmtree(oldroot)
except OSError:
pass
try:
shutil.move(currentroot, oldroot)
os.unlink(linkroot)
os.symlink(oldroot, linkroot)
except OSError:
pass
else:
# This current directory is bad, move it away
shutil.rmtree(currentroot)
# dxr xref files (index + sqlitedb) go in wwwdir/treename-current/.dxr_xref
# and we'll symlink it to wwwdir/treename later
htmlroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
dbdir = os.path.join(htmlroot, '.dxr_xref')
os.makedirs(dbdir, 0755)
dbname = treecfg.tree + '.sqlite'
retcode = 0
if doxref:
builddb(treecfg, dbdir)
if treecfg.isdblive:
f = open(os.path.join(dbdir, '.success'), 'w')
f.close()
elif treecfg.isdblive:
# If the database is live, we need to copy database info from the old
# version of the code
oldhtml = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
olddbdir = os.path.join(oldhtml, '.dxr_xref')
shutil.rmtree(dbdir)
shutil.copytree(olddbdir, dbdir)
# Build static html
if dohtml:
if not doxref:
big_blob = dxr.load_big_blob(treecfg)
# Do we need to do file pivoting?
for plugin in dxr.get_active_plugins(treecfg):
if plugin.__name__ in big_blob:
plugin.pre_html_process(treecfg, big_blob[plugin.__name__])
dxr.htmlbuilders.build_htmlifier_map(dxr.get_active_plugins(treecfg))
treecfg.database = os.path.join(dbdir, dbname)
n = cpu_count()
p = Pool(processes=n)
print 'Building HTML files for %s...' % treecfg.tree
debug = (debugfile is not None)
index_list = open(os.path.join(dbdir, "file_list.txt"), 'w')
file_list = []
def getOutputFiles():
for regular in treecfg.getFileList():
yield regular
filelist = set()
for plug in big_blob:
try:
filelist.update(big_blob[plug]["byfile"].keys())
except KeyError:
pass
for filename in filelist:
if filename.startswith("--GENERATED--/"):
relpath = filename[len("--GENERATED--/"):]
yield filename, os.path.join(treecfg.objdir, relpath)
if debugfile:
output_files = glob.glob (treecfg.sourcedir + '/' + debugfile)
if output_files == []:
print 'Error: Glob %s doesn\'t match any files' % debugfile
sys.exit (1)
last_dir = None
for f in getOutputFiles():
# In debug mode, we only care about some files
if debugfile and not treecfg.sourcedir + '/' + f[0] in output_files: continue
index_list.write(f[0] + '\n')
cpypath = os.path.join(htmlroot, f[0])
srcpath = f[1]
file_list.append(f)
# Make output directory
cpydir = os.path.dirname(cpypath)
if not os.path.exists(cpydir):
os.makedirs(cpydir)
p.apply_async(async_toHTML, [treecfg, srcpath, cpypath + ".html"])
if file_list == []:
print 'Error: No files found to index'
sys.exit (0)
p.apply_async(make_index, [file_list, dbdir])
index_list.close()
p.close()
p.join()
# Generate index.html files
# XXX: This wants to be parallelized. However, I seem to run into problems
# if it isn't.
def genhtml(treecfg, dirname, fnames):
make_index_html(treecfg, dirname, fnames, htmlroot)
os.path.walk(htmlroot, genhtml, treecfg)
# If the database is live, we need to switch the live to the new version
if treecfg.isdblive:
try:
os.unlink(linkroot)
shutil.rmtree(oldroot)
except OSError:
pass
os.symlink(currentroot, linkroot)
def indextree(treecfg, doxref, dohtml, debugfile):
global big_blob
# dxr xref files (index + sqlitedb) go in wwwdir/treename-current/.dxr_xref
# and we'll symlink it to wwwdir/treename later
tmproot = tempfile.mkdtemp(prefix=(os.path.join(treecfg.wwwdir, '.' +
treecfg.tree + '.')))
linkroot = os.path.join(treecfg.wwwdir, treecfg.tree)
dbdir = os.path.join(tmproot, '.dxr_xref')
os.makedirs(dbdir, 0755)
dbname = treecfg.tree + '.sqlite'
retcode = 0
if doxref:
builddb(treecfg, dbdir, tmproot)
# Build static html
if dohtml:
# if not doxref:
# big_blob = dxr.load_big_blob(treecfg, tmproot)
# Do we need to do file pivoting?
for plugin in dxr.get_active_plugins(treecfg):
if plugin.__name__ in big_blob:
plugin.pre_html_process(treecfg, big_blob[plugin.__name__])
dxr.htmlbuilders.build_htmlifier_map(dxr.get_active_plugins(treecfg))
treecfg.database = os.path.join(dbdir, dbname)
n = cpu_count()
p = Pool(processes=n)
print 'Building HTML files for %s...' % treecfg.tree
debug = (debugfile is not None)
index_list = open(os.path.join(dbdir, "file_list.txt"), 'w')
file_list = []
def getOutputFiles(conn):
for regular in treecfg.getFileList():
yield regular
for row in conn.execute(
"SELECT path FROM files WHERE path LIKE '--GENERATED--/%'"
).fetchall():
filename = row[0]
relpath = filename[len('--GENERATED--/'):]
yield (filename, os.path.join(treecfg.objdir, relpath), row[0])
if debugfile:
output_files = glob.glob(treecfg.sourcedir + '/' + debugfile)
if output_files == []:
print 'Error: Glob %s doesn\'t match any files' % debugfile
sys.exit(1)
last_dir = None
conn = getdbconn(treecfg, dbdir)
for f in getOutputFiles(conn):
# In debug mode, we only care about some files
if debugfile and not treecfg.sourcedir + '/' + f[0] in output_files:
continue
index_list.write(f[0] + '\n')
cpypath = os.path.join(tmproot, f[0])
srcpath = f[1]
file_list.append(f)
if len(f) > 2:
dbpath = f[2]
else:
dbpath = None
# Make output directory
cpydir = os.path.dirname(cpypath)
if not os.path.exists(cpydir):
os.makedirs(cpydir)
def is_text(srcpath):
# xdg.Mime considers .lo as text, which is technically true but useless
if srcpath[-3:] == '.lo': return False
import xdg.Mime
mimetype = str(xdg.Mime.get_type(srcpath))
for valid in [
'text', 'xml', 'shellscript', 'perl', 'm4', 'xbel',
'javascript'
]:
if valid in mimetype:
return True
# Force indexing of nsis files
if srcpath[-4:] == '.nsh' or srcpath[-4:] == '.nsi':
return True
return False
if not is_text(srcpath):
continue
# p.apply_async(async_toHTML, [treecfg, srcpath, cpypath + ".html", dbdir])
try:
dxr.htmlbuilders.make_html(srcpath, cpypath + ".html", treecfg,
big_blob, conn, dbpath)
except Exception, e:
print 'Error on file %s:' % srcpath
import traceback
traceback.print_exc()
if file_list == []:
print 'Error: No files found to index'
sys.exit(0)
p.apply_async(make_index, [file_list, dbdir, treecfg])
index_list.close()
p.close()
p.join()
# Generate index.html files
# XXX: This wants to be parallelized. However, I seem to run into problems
# if it isn't.
def genhtml(treecfg, dirname, fnames):
make_index_html(treecfg, dirname, fnames, tmproot)
os.path.walk(tmproot, genhtml, treecfg)
def indextree(treecfg, doxref, dohtml, debugfile):
global big_blob
# dxr xref files (index + sqlitedb) go in wwwdir/treename-current/.dxr_xref
# and we'll symlink it to wwwdir/treename later
tmproot = tempfile.mkdtemp(prefix = (os.path.join(treecfg.wwwdir, '.' + treecfg.tree + '.')))
linkroot = os.path.join(treecfg.wwwdir, treecfg.tree)
dbdir = os.path.join(tmproot, '.dxr_xref')
os.makedirs(dbdir, 0755)
dbname = treecfg.tree + '.sqlite'
retcode = 0
if doxref:
builddb(treecfg, dbdir, tmproot)
# Build static html
if dohtml:
# if not doxref:
# big_blob = dxr.load_big_blob(treecfg, tmproot)
# Do we need to do file pivoting?
for plugin in dxr.get_active_plugins(treecfg):
if plugin.__name__ in big_blob:
plugin.pre_html_process(treecfg, big_blob[plugin.__name__])
dxr.htmlbuilders.build_htmlifier_map(dxr.get_active_plugins(treecfg))
treecfg.database = os.path.join(dbdir, dbname)
n = cpu_count()
p = Pool(processes=n)
print 'Building HTML files for %s...' % treecfg.tree
debug = (debugfile is not None)
index_list = open(os.path.join(dbdir, "file_list.txt"), 'w')
file_list = []
def getOutputFiles(conn):
for regular in treecfg.getFileList():
yield regular
for row in conn.execute("SELECT path FROM files WHERE path LIKE '--GENERATED--/%'").fetchall():
filename = row[0]
relpath = filename[len('--GENERATED--/'):]
yield (filename, os.path.join(treecfg.objdir, relpath), row[0])
if debugfile:
output_files = glob.glob (treecfg.sourcedir + '/' + debugfile)
if output_files == []:
print 'Error: Glob %s doesn\'t match any files' % debugfile
sys.exit (1)
last_dir = None
conn = getdbconn(treecfg, dbdir)
for f in getOutputFiles(conn):
# In debug mode, we only care about some files
if debugfile and not treecfg.sourcedir + '/' + f[0] in output_files: continue
index_list.write(f[0] + '\n')
cpypath = os.path.join(tmproot, f[0])
srcpath = f[1]
file_list.append(f)
if len(f) > 2:
dbpath = f[2]
else:
dbpath = None
# Make output directory
cpydir = os.path.dirname(cpypath)
if not os.path.exists(cpydir):
os.makedirs(cpydir)
def is_text(srcpath):
# xdg.Mime considers .lo as text, which is technically true but useless
if srcpath[-3:] == '.lo': return False
import xdg.Mime
mimetype = str(xdg.Mime.get_type (srcpath))
for valid in ['text', 'xml', 'shellscript', 'perl', 'm4', 'xbel', 'javascript']:
if valid in mimetype:
return True
# Force indexing of nsis files
if srcpath[-4:] == '.nsh' or srcpath[-4:] == '.nsi':
return True
return False
if not is_text(srcpath):
continue
# p.apply_async(async_toHTML, [treecfg, srcpath, cpypath + ".html", dbdir])
try:
dxr.htmlbuilders.make_html(srcpath, cpypath + ".html", treecfg, big_blob, conn, dbpath)
except Exception, e:
print 'Error on file %s:' % srcpath
import traceback
traceback.print_exc()
if file_list == []:
print 'Error: No files found to index'
sys.exit (0)
p.apply_async(make_index, [file_list, dbdir, treecfg])
index_list.close()
p.close()
p.join()
# Generate index.html files
# XXX: This wants to be parallelized. However, I seem to run into problems
# if it isn't.
def genhtml(treecfg, dirname, fnames):
make_index_html(treecfg, dirname, fnames, tmproot)
os.path.walk(tmproot, genhtml, treecfg)
def indextree(treecfg, doxref, dohtml, debugfile):
global big_blob
# If we're live, we'll need to move -current to -old; we'll move it back
# after we're done.
if treecfg.isdblive:
currentroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
oldroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
linkroot = os.path.join(treecfg.wwwdir, treecfg.tree)
if os.path.isdir(currentroot):
if os.path.exists(os.path.join(currentroot, '.dxr_xref', '.success')):
# Move current -> old, change link to old
try:
shutil.rmtree(oldroot)
except OSError:
assert_permission_denied()
pass
try:
shutil.move(currentroot, oldroot)
os.unlink(linkroot)
os.symlink(oldroot, linkroot)
except OSError:
assert_permission_denied()
pass
else:
# This current directory is bad, move it away
shutil.rmtree(currentroot)
# dxr xref files (index + sqlitedb) go in wwwdir/treename-current/.dxr_xref
# and we'll symlink it to wwwdir/treename later
htmlroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
dbdir = os.path.join(htmlroot, '.dxr_xref')
os.makedirs(dbdir, 0755)
dbname = dxr.get_database_filename(treecfg)
retcode = 0
if doxref:
builddb(treecfg, dbdir)
if treecfg.isdblive:
f = open(os.path.join(dbdir, '.success'), 'w')
f.close()
elif treecfg.isdblive:
# If the database is live, we need to copy database info from the old
# version of the code
oldhtml = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
olddbdir = os.path.join(oldhtml, '.dxr_xref')
shutil.rmtree(dbdir)
shutil.copytree(olddbdir, dbdir)
# Build static html
if dohtml:
if not doxref:
big_blob = dxr.load_big_blob(treecfg)
# Do we need to do file pivoting?
for plugin in dxr.get_active_plugins(treecfg):
if plugin.__name__ in big_blob:
plugin.pre_html_process(treecfg, big_blob[plugin.__name__])
dxr.htmlbuilders.build_htmlifier_map(dxr.get_active_plugins(treecfg))
treecfg.database = dbname
n = cpu_count()
p = Pool(processes=n)
print 'Building HTML files for %s...' % treecfg.tree, ; sys.stdout.flush ()
index_list = open(os.path.join(dbdir, "file_list.txt"), 'w')
file_list = []
def getOutputFiles():
for regular in treecfg.getFileList():
yield regular
filelist = set()
for plug in big_blob:
try:
filelist.update(big_blob[plug]["byfile"].keys())
except KeyError:
pass
for filename in filelist:
if filename.startswith("--GENERATED--/"):
relpath = filename[len("--GENERATED--/"):]
yield filename, os.path.join(treecfg.objdir, relpath)
if debugfile:
output_files = glob.glob (treecfg.sourcedir + '/' + debugfile)
if output_files == []:
print 'Error: Glob %s doesn\'t match any files' % debugfile
sys.exit (1)
last_dir = None
for f in getOutputFiles():
# In debug mode, we only care about some files
if debugfile and not treecfg.sourcedir + '/' + f[0] in output_files: continue
if os.path.dirname (treecfg.sourcedir + '/' + f[0]) != last_dir:
last_dir = os.path.dirname (treecfg.sourcedir + '/' + f[0])
print os.path.basename (last_dir), ; sys.stdout.flush ()
index_list.write(f[0] + '\n')
cpypath = os.path.join(htmlroot, f[0])
srcpath = f[1]
file_list.append(f)
# Make output directory
cpydir = os.path.dirname(cpypath)
if not os.path.exists(cpydir):
os.makedirs(cpydir)
def is_text(srcpath):
# xdg.Mime considers .lo as text, which is technically true but useless
if srcpath[-3:] == '.lo': return False
import xdg.Mime
mimetype = str(xdg.Mime.get_type (srcpath))
for valid in ['text', 'xml', 'shellscript', 'perl', 'm4', 'xbel']:
if valid in mimetype:
return True
return False
if not is_text(srcpath):
continue
p.apply_async(async_toHTML, [treecfg, srcpath, cpypath + ".html"])
if file_list == []:
print 'Error: No files found to index'
sys.exit (0)
print '...Done', '%d files indexed' % len (file_list)
p.apply_async(make_index, [file_list, dbdir])
index_list.close()
p.close()
p.join()
# Generate index.html files
# XXX: This wants to be parallelized. However, I seem to run into problems
# if it isn't.
def genhtml(treecfg, dirname, fnames):
make_index_html(treecfg, dirname, fnames, htmlroot)
os.path.walk(htmlroot, genhtml, treecfg)
# If the database is live, we need to switch the live to the new version
if treecfg.isdblive:
try:
try:
shutil.rmtree(oldroot)
except:
pass
try:
os.remove(linkroot + '.tmp')
except:
pass
os.symlink(currentroot, linkroot + '.tmp')
os.rename(linkroot + '.tmp', linkroot)
except OSError:
msg = sys.exc_info()[1] # Python 2/3 compatibility
print ('Replacing database failed: %s' % str(msg))