def indextree(treecfg, doxref, dohtml, debugfile):
global big_blob
# If we're live, we'll need to move -current to -old; we'll move it back
# after we're done.
if treecfg.isdblive:
currentroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
oldroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
linkroot = os.path.join(treecfg.wwwdir, treecfg.tree)
if os.path.isdir(currentroot):
if os.path.exists(os.path.join(currentroot, '.dxr_xref', '.success')):
# Move current -> old, change link to old
try:
shutil.rmtree(oldroot)
except OSError:
assert_permission_denied()
pass
try:
shutil.move(currentroot, oldroot)
os.unlink(linkroot)
os.symlink(oldroot, linkroot)
except OSError:
assert_permission_denied()
pass
else:
# This current directory is bad, move it away
shutil.rmtree(currentroot)
# dxr xref files (index + sqlitedb) go in wwwdir/treename-current/.dxr_xref
# and we'll symlink it to wwwdir/treename later
htmlroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
dbdir = os.path.join(htmlroot, '.dxr_xref')
os.makedirs(dbdir, 0755)
dbname = dxr.get_database_filename(treecfg)
retcode = 0
if doxref:
builddb(treecfg, dbdir)
if treecfg.isdblive:
f = open(os.path.join(dbdir, '.success'), 'w')
f.close()
elif treecfg.isdblive:
# If the database is live, we need to copy database info from the old
# version of the code
oldhtml = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
olddbdir = os.path.join(oldhtml, '.dxr_xref')
shutil.rmtree(dbdir)
shutil.copytree(olddbdir, dbdir)
# Build static html
if dohtml:
if not doxref:
big_blob = dxr.load_big_blob(treecfg)
# Do we need to do file pivoting?
for plugin in dxr.get_active_plugins(treecfg):
if plugin.__name__ in big_blob:
plugin.pre_html_process(treecfg, big_blob[plugin.__name__])
dxr.htmlbuilders.build_htmlifier_map(dxr.get_active_plugins(treecfg))
treecfg.database = dbname
n = cpu_count()
p = Pool(processes=n)
print 'Building HTML files for %s...' % treecfg.tree, ; sys.stdout.flush ()
index_list = open(os.path.join(dbdir, "file_list.txt"), 'w')
file_list = []
def getOutputFiles():
for regular in treecfg.getFileList():
yield regular
filelist = set()
for plug in big_blob:
try:
filelist.update(big_blob[plug]["byfile"].keys())
except KeyError:
pass
for filename in filelist:
if filename.startswith("--GENERATED--/"):
relpath = filename[len("--GENERATED--/"):]
yield filename, os.path.join(treecfg.objdir, relpath)
if debugfile:
output_files = glob.glob (treecfg.sourcedir + '/' + debugfile)
if output_files == []:
print 'Error: Glob %s doesn\'t match any files' % debugfile
sys.exit (1)
last_dir = None
for f in getOutputFiles():
# In debug mode, we only care about some files
if debugfile and not treecfg.sourcedir + '/' + f[0] in output_files: continue
if os.path.dirname (treecfg.sourcedir + '/' + f[0]) != last_dir:
last_dir = os.path.dirname (treecfg.sourcedir + '/' + f[0])
print os.path.basename (last_dir), ; sys.stdout.flush ()
index_list.write(f[0] + '\n')
cpypath = os.path.join(htmlroot, f[0])
srcpath = f[1]
file_list.append(f)
# Make output directory
cpydir = os.path.dirname(cpypath)
if not os.path.exists(cpydir):
os.makedirs(cpydir)
def is_text(srcpath):
# xdg.Mime considers .lo as text, which is technically true but useless
if srcpath[-3:] == '.lo': return False
import xdg.Mime
mimetype = str(xdg.Mime.get_type (srcpath))
for valid in ['text', 'xml', 'shellscript', 'perl', 'm4', 'xbel']:
if valid in mimetype:
return True
return False
if not is_text(srcpath):
continue
p.apply_async(async_toHTML, [treecfg, srcpath, cpypath + ".html"])
if file_list == []:
print 'Error: No files found to index'
sys.exit (0)
print '...Done', '%d files indexed' % len (file_list)
p.apply_async(make_index, [file_list, dbdir])
index_list.close()
p.close()
p.join()
# Generate index.html files
# XXX: This wants to be parallelized. However, I seem to run into problems
# if it isn't.
def genhtml(treecfg, dirname, fnames):
make_index_html(treecfg, dirname, fnames, htmlroot)
os.path.walk(htmlroot, genhtml, treecfg)
# If the database is live, we need to switch the live to the new version
if treecfg.isdblive:
try:
try:
shutil.rmtree(oldroot)
except:
pass
try:
os.remove(linkroot + '.tmp')
except:
pass
os.symlink(currentroot, linkroot + '.tmp')
os.rename(linkroot + '.tmp', linkroot)
except OSError:
msg = sys.exc_info()[1] # Python 2/3 compatibility
print ('Replacing database failed: %s' % str(msg))
def indextree(treecfg, doxref, dohtml, debugfile):
global big_blob
# If we're live, we'll need to move -current to -old; we'll move it back
# after we're done.
if treecfg.isdblive:
currentroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
oldroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
linkroot = os.path.join(treecfg.wwwdir, treecfg.tree)
if os.path.isdir(currentroot):
if os.path.exists(os.path.join(currentroot, '.dxr_xref', '.success')):
# Move current -> old, change link to old
try:
shutil.rmtree(oldroot)
except OSError:
pass
try:
shutil.move(currentroot, oldroot)
os.unlink(linkroot)
os.symlink(oldroot, linkroot)
except OSError:
pass
else:
# This current directory is bad, move it away
shutil.rmtree(currentroot)
# dxr xref files (index + sqlitedb) go in wwwdir/treename-current/.dxr_xref
# and we'll symlink it to wwwdir/treename later
htmlroot = os.path.join(treecfg.wwwdir, treecfg.tree + '-current')
dbdir = os.path.join(htmlroot, '.dxr_xref')
os.makedirs(dbdir, 0755)
dbname = treecfg.tree + '.sqlite'
retcode = 0
if doxref:
builddb(treecfg, dbdir)
if treecfg.isdblive:
f = open(os.path.join(dbdir, '.success'), 'w')
f.close()
elif treecfg.isdblive:
# If the database is live, we need to copy database info from the old
# version of the code
oldhtml = os.path.join(treecfg.wwwdir, treecfg.tree + '-old')
olddbdir = os.path.join(oldhtml, '.dxr_xref')
shutil.rmtree(dbdir)
shutil.copytree(olddbdir, dbdir)
# Build static html
if dohtml:
if not doxref:
big_blob = dxr.load_big_blob(treecfg)
# Do we need to do file pivoting?
for plugin in dxr.get_active_plugins(treecfg):
if plugin.__name__ in big_blob:
plugin.pre_html_process(treecfg, big_blob[plugin.__name__])
dxr.htmlbuilders.build_htmlifier_map(dxr.get_active_plugins(treecfg))
treecfg.database = os.path.join(dbdir, dbname)
n = cpu_count()
p = Pool(processes=n)
print 'Building HTML files for %s...' % treecfg.tree
debug = (debugfile is not None)
index_list = open(os.path.join(dbdir, "file_list.txt"), 'w')
file_list = []
def getOutputFiles():
for regular in treecfg.getFileList():
yield regular
filelist = set()
for plug in big_blob:
try:
filelist.update(big_blob[plug]["byfile"].keys())
except KeyError:
pass
for filename in filelist:
if filename.startswith("--GENERATED--/"):
relpath = filename[len("--GENERATED--/"):]
yield filename, os.path.join(treecfg.objdir, relpath)
if debugfile:
output_files = glob.glob (treecfg.sourcedir + '/' + debugfile)
if output_files == []:
print 'Error: Glob %s doesn\'t match any files' % debugfile
sys.exit (1)
last_dir = None
for f in getOutputFiles():
# In debug mode, we only care about some files
if debugfile and not treecfg.sourcedir + '/' + f[0] in output_files: continue
index_list.write(f[0] + '\n')
cpypath = os.path.join(htmlroot, f[0])
srcpath = f[1]
file_list.append(f)
# Make output directory
cpydir = os.path.dirname(cpypath)
if not os.path.exists(cpydir):
os.makedirs(cpydir)
p.apply_async(async_toHTML, [treecfg, srcpath, cpypath + ".html"])
if file_list == []:
print 'Error: No files found to index'
sys.exit (0)
p.apply_async(make_index, [file_list, dbdir])
index_list.close()
p.close()
p.join()
# Generate index.html files
# XXX: This wants to be parallelized. However, I seem to run into problems
# if it isn't.
def genhtml(treecfg, dirname, fnames):
make_index_html(treecfg, dirname, fnames, htmlroot)
os.path.walk(htmlroot, genhtml, treecfg)
# If the database is live, we need to switch the live to the new version
if treecfg.isdblive:
try:
os.unlink(linkroot)
shutil.rmtree(oldroot)
except OSError:
pass
os.symlink(currentroot, linkroot)
sys.path.append('..')
import ConfigParser
import imp, dxr
# Read in the configuration for the test
tests = ConfigParser.ConfigParser()
tests.read('tests.ini')
testname = sys.argv[1]
# Get the database data for the tree
dxrcfg = dxr.load_config(tests.get(testname, 'dxrconfig'))
if len(dxrcfg.trees) != 1:
raise Exception('Database can only have one tree')
tree = dxrcfg.trees[0]
big_blob = dxr.load_big_blob(tree)
# For the purposes of language data, dxr.languages.language_data too
big_blob['dxr.language_data'] = dxr.languages.language_data
# Load the checker data
load_data = open(tests.get(testname, 'checkdb'))
check_blob = eval(load_data.read())
load_data.close()
errors = []
def verifyDatabase(output, standard, should_be_present, string):
for tablename, table in standard.iteritems():
sub_present = should_be_present
if isinstance(tablename, str) and tablename[0] == '~':
