def main(sxwfile):
if not sxwfile.endswith('manual.sxw'):
return
left, right = os.path.split(sxwfile)
if left:
left, right = os.path.split(left)
if not right == '_genesis':
return
if not left:
return
left, right = os.path.split(left)
finalsourcedir = left.replace('\\', '/')
finaldocumentationdir = finalsourcedir
repositoryname = os.path.split(
os.path.split(finaldocumentationdir)[0])[1].replace('\\', '/')
finalbuilddir = ospj(left, '_make', 'build').replace('\\', '/')
finalnotversioneddir = ospj(left, '_not_versioned').replace('\\', '/')
sxwfile = sxwfile.replace('\\', '/')
destdir = '/'.join(sxwfile.split('/')[:-1]).replace('\\', '/')
print
if 1 or printfilenames:
print 'repositoryname :', repositoryname
print 'finalsourcedir :', finalsourcedir
print 'finalbuilddir :', finalbuilddir
print 'finalnotversioneddir:', finalnotversioneddir
print 'destdir :', destdir
if not mockup_uno:
# sxw -> html
srcfile = sxwfile
destfile = sxwfile[:-4] + '.html'
if 1 and printfilenames:
print 1, srcfile
print 2, destfile
else:
print repositoryname, '/'.join(srcfile.split('/')[-2:])
print repositoryname, '/'.join(destfile.split('/')[-2:])
if 1:
error = convertsxw2html(srcfile, destfile)
if error:
print error
# html -> cleaned
srcfile = sxwfile[:-4] + '.html'
destfile = sxwfile[:-4] + '-cleaned.html'
if 1 and printfilenames:
print 1, srcfile
print 2, destfile
else:
print repositoryname, '/'.join(destfile.split('/')[-2:])
if 1:
copyclean.main(srcfile, destfile)
# cleaned -> from-tidy
srcfile = sxwfile[:-4] + '-cleaned.html'
destfile = sxwfile[:-4] + '-from-tidy.html'
if 1 and printfilenames:
print 1, srcfile
print 2, destfile
else:
print repositoryname, '/'.join(destfile.split('/')[-2:])
if 1:
errorfile = ospj(destdir, 'tidy-error-log.txt')
cmd = ' '.join([
'tidy', '-asxhtml', '-utf8', '-f', errorfile, '-o', destfile,
srcfile
])
if os.path.exists(destfile):
os.remove(destfile)
returncode = subprocess.call(cmd, shell=True)
# manual-from-tidy.html -> manual.rst
# python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt
srcfile = ospj(destdir, 'manual-from-tidy.html').replace('\\', '/')
destfile = ospj(destdir, 'manual.rst').replace('\\', '/')
treefile = f3name = ospj(destdir, 'restparser-tree.txt')
logfile = f4name = ospj(destdir, 'restparser-log.txt')
if 1 and printfilenames:
print 1, srcfile
print 2, destfile
else:
print repositoryname, '/'.join(destfile.split('/')[-2:])
if 1:
if ospe(destfile):
os.remove(destfile)
# ooxhtml2rst.main(f1name, f2name, f3name=None, f4name=None, appendlog=0, taginfo=0):
ooxhtml2rst.main(srcfile, destfile, treefile, logfile)
# normalize_empty_lines
srcfile = ospj(destdir, 'manual.rst').replace('\\', '/')
destfile = ospj(destdir,
'manual-empty-lines-normalized.rst').replace('\\', '/')
if 1 and printfilenames:
print 1, srcfile
print 2, destfile
else:
print repositoryname, '/'.join(destfile.split('/')[-2:])
if 1:
normalize_empty_lines.main(srcfile, destfile, 2)
os.remove(srcfile)
os.rename(destfile, srcfile)
# first slicing
srcfile = ospj(destdir, 'manual.rst').replace('\\', '/')
tempdir = ospj(destdir, 'temp').replace('\\', '/')
if 1 and printfilenames:
print 1, srcfile
print 2, tempdir
else:
print repositoryname, '/'.join(tempdir.split('/')[-2:])
if 1:
slice_to_numbered_files.main(srcfile, tempdir)
# final file structure
print repositoryname, '/'.join(finalsourcedir.split('/')[-2:])
srcdirimages = destdir
if 1 and printfilenames:
print 1, tempdir
print 2, finalsourcedir
print 3, srcdirimages
if 1:
write_sphinx_structure.main(tempdir,
finalsourcedir,
srcdirimages,
verbose=0)
# res files
destdir = ospj(finalsourcedir, '_make')
if not ospe(destdir):
os.makedirs(destdir)
files = os.listdir(resdir)
for afile in files:
if afile in [
'default_conf.py',
'default_make.bat',
'default_make-html.bat',
'default_Makefile',
]:
destname = afile[len('default_'):]
destpath = ospj(destdir, destname)
if not ospe(destpath):
srcpath = ospj(resdir, afile)
shutil.copyfile(srcpath, destpath)
if not ospe(finalbuilddir):
os.makedirs(finalbuilddir)
# .gitignore
srcpath = ospj(resdir, 'default_gitignore_toplevel')
destpath = ospj(finaldocumentationdir, '.gitignore')
if not ospe(destpath):
shutil.copyfile(srcpath, destpath)
# .gitignore
srcpath = ospj(resdir, 'default_gitignore_all_in_this_folder')
destpath = ospj(finalbuilddir, '.gitignore')
if not ospe(destpath):
shutil.copyfile(srcpath, destpath)
# .gitignore
srcpath = ospj(resdir, 'default_gitignore_all_in_this_folder')
destpath = ospj(finalnotversioneddir, '.gitignore')
if not ospe(destpath):
shutil.copyfile(srcpath, destpath)
# more
# place warnings.txt there for illustration
f2path = ospj(finalnotversioneddir, 'warnings.txt')
if not ospe(f2path):
f2 = file(f2path, 'w')
f2.close()
print repositoryname, 'done.'
return
def work(self):
retCode = 0
msg = ''
if self.args.outfilename:
if self.f2name != self.args.outfilename:
return 1, "Specify outfilename without path. Found: '%s'" % self.args.outfilename
if self.f2ext != '.html':
return 1, "Specify 'outfilename' with '.html'. Found: '%s'" % self.args.outfilename
try:
f1 = file(self.args.infile)
except:
retCode, msg = 1, "Cannot read file '%s'" % self.args.infile
return retCode, msg
if not os.path.isdir(self.args.tempdir):
retCode, msg = (1, "Cannot find tempdir '%s'" % self.args.tempdir)
return retCode, msg
if os.path.isdir(self.safetempdir):
try:
shutil.rmtree(self.safetempdir)
except:
retCode, msg = (1, "Cannot remove safetempdir '%s'" %
self.safetempdir)
return retCode, msg
if os.path.isdir(self.safetempdir):
retCode, msg = (1, "Cannot remove safetempdir '%s'" %
self.safetempdir)
return retCode, msg
else:
try:
os.makedirs(self.safetempdir)
except:
retCode, msg = (1, "Cannot create safetempdir '%s'" %
self.safetempdir)
return retCode, msg
if self.logdir == ospj(self.safetempdir, 'logs'):
os.makedirs(self.logdir)
else:
if not os.path.isdir(self.logdir):
return 1, "Cannot find logdir '%s'" % self.logdir
try:
f2path = ospj(self.logdir, 'test.txt')
f2 = file(f2path, 'w')
f2.close()
os.unlink(f2path)
except IOError:
return 1, "Cannot write to file in logdir: '%s'" % f2path
# let's check the manual.sxw is not corrupted in some way
tested = False
try:
iszipfile = zipfile.is_zipfile(self.f1path)
tested = True
except:
pass
if not tested:
retCode, msg = (1, "Cannot test '%s' for being a zipfile" %
self.f1path)
return retCode, msg
elif not iszipfile:
retCode, msg = (1, "'%s' is not a zipfile" % self.f1path)
return retCode, msg
else:
pass
error = None
try:
zf = zipfile.ZipFile(self.f1path)
testresult = zf.testzip()
error = None
except IOError:
error = 'IOError'
except:
error = 'Unknown Error'
if error:
retCode, msg = (1, "'%s' looks like a zipfile but could not "
"be tested with zipfiletest" % self.f1path)
return retCode, msg
if testresult:
retCode, msg = (1, "'%s' is a zipfile but not legal in some "
"aspects" % self.f1path)
retCode, msg = convertsxw2html(self.f1path, self.f2path)
if retCode:
return retCode, msg
if 0:
cmd = 'chmod +r ' + ospj(self.safetempdir, '*')
subprocess.call(cmd, shell=True)
if 1 and "convert *.gif to *.gif.png":
L = []
dirname = os.path.dirname(self.f2path)
for fname in os.listdir(dirname):
if fname.lower().startswith(
'manual_html_') and fname.lower().endswith('.gif'):
L.append(fname)
if L:
for fname in L:
gifFile = ospj(dirname, fname)
im = PIL.Image.open(gifFile)
pngFile = gifFile + '.png'
im.save(pngFile)
f1 = file(self.f2path)
data = f1.read()
f1.close()
for fname in L:
data = data.replace(fname, fname + '.png')
f2 = file(self.f2path, "w")
f2.write(data)
f2.close()
if 1:
try:
copyclean.main(self.f2path, self.f2path_cleaned)
except:
retCode, msg = (1, "Cannot run 'copyclean.main()")
return retCode, msg
if 1:
# manual-cleaned.html -> manual-from-tidy.html
# # step: Use tidy to convert from HTML-4 to XHTML
# tidy -asxhtml -utf8 -f $EXTENSIONS/$EXTKEY/nightly/tidy-error-log.txt -o $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/1-cleaned.html
cmd = ' '.join([
'tidy', '-asxhtml', '-utf8', '-f', self.f2path_tidy_error_log,
'-o', self.f2path_from_tidy, self.f2path_cleaned
])
retCode = subprocess.call(cmd, shell=True)
if not ospe(self.f2path_from_tidy):
retCode, msg = (1,
"Cannot create '%s'" % self.f2path_from_tidy)
if 1:
# parse from *.html to *.rst
arg = Namespace()
arg.infile = self.f2path_from_tidy
arg.outfile = self.f2path_rst
arg.treefile = None # no treefile
arg.treefile = self.f2path_rst_treefile
arg.logfile = None # no logfile
arg.logfile = self.f2path_rst_logfile
arg.appendlog = 0
arg.taginfo = 0
tabletypes = ['t3flt', 'dl', 'flt']
tabletypes = ['t3flt', 'dl']
for i, tablesas in enumerate(tabletypes):
outfile = arg.outfile[:-3] + tablesas + '.rst'
ooxhtml2rst.main(arg.infile, outfile, arg.treefile,
arg.logfile, arg.appendlog, arg.taginfo,
tablesas)
if not os.path.exists(outfile):
return 1, "ooxhtml2rst.main(): could not create ReST file '%s'" % outfile
if i == 0:
self.f2path_rst = outfile
arg.treefile = None
arg.logfile = None
self.rstfilepaths = []
for fname in os.listdir(self.safetempdir):
if fname.endswith('.rst'):
fpath = ospj(self.safetempdir, fname)
if os.path.isfile(fpath):
self.rstfilepaths.append(fpath)
if 1:
# for each of our newly created *.rst provide a Docutils rendering
# errorfilename = 'sxw2html-conversion-error.txt'
# self.t3docutils_stylesheet_path
# self.usr_bin_python
# self.t3rst2html_script
# self.safetempdir
for f2path_rst in self.rstfilepaths:
normalize_empty_lines.main(f2path_rst, self.f2path_rst_temp, 2)
os.remove(f2path_rst)
os.rename(self.f2path_rst_temp, f2path_rst)
if 1:
arg = Namespace()
arg.pathToTemplate = self.t3docutils_template_path
for arg.srcfile in self.rstfilepaths:
arg.basename = os.path.basename(arg.srcfile)
temp = arg.basename
temp = temp.replace('.dl.html', '.html')
temp = temp.replace('.flt.html', '.html')
temp = temp.replace('.t3flt.html', '.html')
arg.basename_generic = temp
arg.destfile = arg.srcfile[:-4] + '.html'
arg.warningsfile = ospj(
self.logdir,
arg.basename_generic + '.t3rst2html-warnings.txt')
cmd = ' '.join([
self.usr_bin_python,
self.t3rst2html_script,
'--source-link',
# '--link-stylesheet',
# '--stylesheet-path=%s' % stylesheet_path,
'--template=%s' % arg.pathToTemplate,
'--field-name-limit=0',
'--warnings=%s' % arg.warningsfile,
arg.srcfile,
arg.destfile,
])
arg.stdout_log = ospj(
self.logdir,
arg.basename_generic + '.t3rst2html-stdout.txt')
arg.stderr_log = ospj(
self.logdir,
arg.basename_generic + '.t3rst2html-stderr-log.txt')
# errors = 'strict', 'replace', 'ignore', 'xmlcharrefreplace', 'backslashreplace'
f2stdout = codecs.open(arg.stdout_log,
'w',
'utf-8',
errors='backslashreplace')
f2stderr = codecs.open(arg.stderr_log,
'w',
'utf-8',
errors='backslashreplace')
# devnull = file('/dev/null', 'w')
returncode = subprocess.call(cmd,
stdout=f2stdout,
stderr=f2stderr,
shell=True)
f2stderr.close()
f2stdout.close()
# returncode = subprocess.call(cmd, shell=True)
# devnull.close()
# ==================================================
# make ./Documentation structure
# --------------------------------------------------
if 1 and 'select the rst file with the desired table handling':
for self.f2path_rst in self.rstfilepaths:
if self.f2path_rst.endswith('.t3flt.rst'):
pass
if self.f2path_rst.endswith('.dl.rst'):
break
else:
self.f2path_rst = None
arg = Namespace()
arg.f2path_rst = self.f2path_rst
arg.safetempdir_sliced = self.safetempdir_sliced
arg.f2path_documentation = self.f2path_documentation
arg.srcdirimages = self.safetempdir
arg.resdir = self.resdir
retCode, msg = write_sphinx_structure.main(arg)
# travel all *.rst files in ./Documentation and
# insert an intersphinx label before each section
# only underlined sections are recognized
prepend_sections_with_labels.main(self.f2path_documentation)
## ####
## Ah, oh, uh, this was a silly idea. Forget it.
##
## # travel all *.rst files in ./Documentation and
## # try to check whether the first row of a .. t3-field-list-table::
## # should be removed. If so do so.
## remove_first_t3fieldlisttable_row.main(self.f2path_documentation)
# travel all *.rst files in ./Documentation and
# and try to tweak the '.. container:: table-row' of 'definition list tables'
tweak_dllisttables.main(self.f2path_documentation)
retCode = 0
msg = "Ok"
return retCode, msg
def walk_ter_manuals_html_to_rest(rootfolder):
prelpath = len(rootfolder)
proceeding = True
cnt = 0
cntnew = 0
for path, dirs, files in os.walk(rootfolder):
proceedwithfile = True
destdir = path
dirs.sort()
if not proceeding:
dirs[:] = []
break
else:
for afile in files:
if not proceeding:
break
if afile == 'manual.html' and len(
os.path.split(path)[1].split('.')) == 3:
left, version = os.path.split(path)
left, extkey = os.path.split(left)
manualhtmlfile = os.path.join(path, 'manual.html')
manualhtmlfile = os.path.join(path, afile)
cleanedfile = os.path.join(destdir, 'manual-cleaned.html')
tidyfile = os.path.join(destdir, 'manual-from-tidy.html')
rstfile = os.path.join(destdir, 'manual.rst')
indexhtmlfile = os.path.join(destdir, 'index.html')
finaldestfile = indexhtmlfile
cnt += 1
if 0 and 'force rebuild':
if os.path.exists(indexhtmlfile):
os.remove(indexhtmlfile)
if 1:
cmd = 'chmod +r ' + os.path.join(destdir, '*')
subprocess.call(cmd, shell=True)
if proceedwithfile and os.path.exists(finaldestfile):
# nothing to do?
print '%04d %4d exists: %s' % (
cnt, cntnew, finaldestfile[prelpath:])
proceedwithfile = False
if proceedwithfile:
# manual.html -> manual-cleaned.html
srcfile = manualhtmlfile
destfile = cleanedfile
print '%04d %4d new: %s' % (cnt, cntnew,
srcfile[prelpath:])
print srcfile
if os.path.exists(destfile):
os.remove(destfile)
proceedwithfile = not os.path.exists(destfile)
if not proceedwithfile:
print 'could not remove %s' % destfile
if proceedwithfile:
copyclean.main(srcfile, destfile)
proceedwithfile = os.path.exists(cleanedfile)
if not proceedwithfile:
print 'could not create %s' % destfile
if proceedwithfile:
# manual-cleaned.html -> manual-from-tidy.html
# # step: Use tidy to convert from HTML-4 to XHTML
# tidy -asxhtml -utf8 -f $EXTENSIONS/$EXTKEY/nightly/tidy-error-log.txt -o $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/1-cleaned.html
srdfile = cleanedfile
destfile = tidyfile
errorfile = os.path.join(destdir, 'tidy-error-log.txt')
cmd = ' '.join([
'tidy', '-asxhtml', '-utf8', '-f', errorfile, '-o',
destfile, cleanedfile
])
if os.path.exists(destfile):
os.remove(destfile)
proceedwithfile = not os.path.exists(destfile)
if not proceedwithfile:
print 'could not remove %s' % destfile
if proceedwithfile:
returncode = subprocess.call(cmd, shell=True)
proceedwithfile = os.path.exists(destfile)
if not proceedwithfile:
print 'could not create %s' % destfile
if proceedwithfile:
# manual-from-tidy.html -> manual.rst
# python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt
srcfile = tidyfile
destfile = rstfile
# treefile = f3name = 'restparser-tree.txt'
treefile = f3name = None
# logfile = f4name = os.path.join(destdir, 'restparser-log.txt')
logfile = f4name = None
if os.path.exists(destfile):
os.remove(destfile)
proceedwithfile = not os.path.exists(destfile)
if not proceedwithfile:
print 'could not remove %s' % destfile
if proceedwithfile:
# ooxhtml2rst.main(f1name, f2name, f3name=None, f4name=None, appendlog=0, taginfo=0):
ooxhtml2rst.main(srcfile, destfile, treefile,
logfile)
proceedwithfile = os.path.exists(destfile)
if not proceedwithfile:
print 'could not create %s' % (destfile, )
if proceedwithfile:
# step: Generate single HTML file from single reST file using the Docutils (no Sphinx involved here)
# rst2htmltypo3.py --link-stylesheet --stylesheet-path=/css/typo3_docutils_styles.css --warnings=$EXTENSIONS/$EXTKEY/nightly/rst2html-warnings.txt $EXTENSIONS/$EXTKEY/nightly/manual.rst $EXTENSIONS/$EXTKEY/nightly/index.html
srcfile = os.path.join(destdir, 'manual.rst')
destfile = os.path.join(destdir, 'index.html')
if proceedwithfile:
# manual.rst -> index.html
srcfile = rstfile
destfile = indexhtmlfile
warningsfile = os.path.join(destdir,
'rst2html-warnings.txt')
if os.path.exists(destfile):
os.remove(destfile)
proceedwithfile = not os.path.exists(destfile)
if not proceedwithfile:
print 'could not remove %s' % destfile
if proceedwithfile:
cmd = ' '.join([
usr_bin_python,
rst2html_script,
'--source-link',
'--link-stylesheet',
'--stylesheet-path=%s' % stylesheet_path,
'--field-name-limit=0',
'--warnings=%s' % warningsfile,
srcfile,
destfile,
])
returncode = subprocess.call(cmd, shell=True)
proceedwithfile = os.path.exists(destfile)
if not proceedwithfile:
print 'could not create %s' % (destfile, )
else:
cntnew += 1
if 0 and cntnew > 100:
proceeding = False
break
print
return proceeding
def extensionProcessorDoAll(extkey, version):
global globalCountExtensions
globalCountExtensions += 1
result = None
destdir = '%s/%s/%s' % (dest_path_to_extensions, extkey, version)
url = "%s/%s/%s/" % (url_of_extensions, extkey, version)
if not ospe(destdir):
os.makedirs(destdir)
srcfile = '%s/manual.sxw' % destdir
if ospe(srcfile):
return result
log.msg('new: %s/%s at: %s' % (extkey, version, url), NL)
wgetsource = 'http://typo3.org/extension-manuals/%s/%s/sxw/?no_cache=1' % (
extkey, version)
cmd = '/usr/bin/wget -q --output-document=%s %s' % (srcfile.replace(
'/', os.sep), wgetsource)
cmd = 'wget -q --output-document=%s %s' % (srcfile.replace(
'/', os.sep), wgetsource)
retCode = subprocess.call(cmd, shell=True)
if retCode != 0:
result = 'wget returned error code %s when getting manual.sxw to: %s' % (
retCode, url)
log.msg(result, NL)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
if not ospe(srcfile):
result = 'no manual.sxw found after wget at: %s' % url
log.msg(result, NL)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
tested = False
try:
iszipfile = zipfile.is_zipfile(srcfile)
tested = True
except:
pass
if not tested:
result = "could not test manual.sxw at: %s" % url
log.msg(result, NL)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
if not iszipfile:
result = "manual.sxw is not a zipfile: %s" % url
log.msg(result, NL)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
error = ''
try:
zf = zipfile.ZipFile(srcfile)
testresult = zf.testzip()
error = None
except IOError:
error = 'IOError'
except:
error = 'Unknown Error'
if error:
result = "manual.sxw looks like a zipfile but the zipfile test gives an '%s' at: %s" % (
error, url)
log.msg(result, NL)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
if testresult:
result = "manual.sxw looks like a zipfile but is somehow damaged at: %s" % url
log.msg(result, NL)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
srcfile = ospj(destdir, 'manual.sxw')
destfile = ospj(destdir, 'manual.html')
error = convertsxw2html(srcfile, destfile)
if error:
result = '%s at: %s' % (error, url)
log.msg(result, NL)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
cmd = 'chmod +r ' + ospj(destdir, '*')
subprocess.call(cmd, shell=True)
srcfile = ospj(destdir, 'manual.html')
destfile = ospj(destdir, 'manual-cleaned.html')
copyclean.main(srcfile, destfile)
# manual-cleaned.html -> manual-from-tidy.html
# # step: Use tidy to convert from HTML-4 to XHTML
# tidy -asxhtml -utf8 -f $EXTENSIONS/$EXTKEY/nightly/tidy-error-log.txt -o $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/1-cleaned.html
srcfile = ospj(destdir, 'manual-cleaned.html')
destfile = ospj(destdir, 'manual-from-tidy.html')
errorfile = ospj(destdir, 'tidy-error-log.txt')
cmd = ' '.join([
'tidy', '-asxhtml', '-utf8', '-f', errorfile, '-o', destfile, srcfile
])
if os.path.exists(destfile):
os.remove(destfile)
returncode = subprocess.call(cmd, shell=True)
if not ospe(destfile):
result = 'could not create %s at %s' % (destfile, url)
log.msg(result, NL)
return result
# manual-from-tidy.html -> manual.rst
# python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt
srcfile = ospj(destdir, 'manual-from-tidy.html')
destfile = ospj(destdir, 'manual.rst')
treefile = f3name = ospj(destdir, 'restparser-tree.txt')
treefile = f3name = None
logfile = f4name = ospj(destdir, 'restparser-log.txt')
logfile = f4name = None
if ospe(destfile):
os.remove(destfile)
# ooxhtml2rst.main(f1name, f2name, f3name=None, f4name=None, appendlog=0, taginfo=0):
ooxhtml2rst.main(srcfile, destfile, treefile, logfile)
if not ospe(destfile):
result = 'could not create %s at %s' % (destfile, url)
log.msg(result, NL)
return result
# step: Generate single HTML file from single reST file using the Docutils (no Sphinx involved here)
# rst2htmltypo3.py --link-stylesheet --stylesheet-path=/css/typo3_docutils_styles.css --warnings=$EXTENSIONS/$EXTKEY/nightly/rst2html-warnings.txt $EXTENSIONS/$EXTKEY/nightly/manual.rst $EXTENSIONS/$EXTKEY/nightly/index.html
srcfile = ospj(destdir, 'manual.rst')
destfile = ospj(destdir, 'index.html')
warningsfile = ospj(destdir, 'rst2html-warnings.txt')
if os.path.exists(destfile):
os.remove(destfile)
cmd = ' '.join([
usr_bin_python,
rst2html_script,
'--source-link',
# '--link-stylesheet',
# '--stylesheet-path=%s' % stylesheet_path,
'--template=%s' % absPathToTemplate,
'--field-name-limit=0',
'--warnings=%s' % warningsfile,
srcfile,
destfile,
])
devnull = file('/dev/null', 'w')
#returncode = subprocess.call(cmd, stdout=devnull, stderr=devnull, shell=True)
returncode = subprocess.call(cmd, shell=True)
devnull.close()
if not ospe(destfile):
result = 'could not create index.html from manual.rst at %s' % url
log.msg(result, NL)
return result
return result
def extensionProcessor(extkey, version):
global globalCountExtensions
globalCountExtensions += 1
result = None
destdir = '%s/%s/%s' % (dest_path_to_extensions, extkey, version)
url = "%s/%s/%s/" % (url_of_extensions, extkey, version)
typo3orgurl = 'http://typo3.org/extensions/repository/view/%s/' % extkey
manual_is_not_available_fpath = destdir + '/manual-is-not-available.txt'
if not ospe(destdir):
os.makedirs(destdir)
destdir_ctime = os.stat(destdir).st_ctime
srcfile = '%s/manual.sxw' % destdir
if 0 and "retry those having 'manual-is-not-available.txt'":
flagfile = destdir + '/manual-is-not-available.txt'
if ospe(srcfile) and ospe(manual_is_not_available_fpath):
os.remove(srcfile)
os.remove(manual_is_not_available_fpath)
if ospe(srcfile) and ospe(ospj(destdir, 'manual.html')):
return result
action = 'new:'
if ospe(manual_is_not_available_fpath):
# retry for a while
maxdelta = 86400 * 1.5
if ospe(srcfile):
# 1350983645 = time.time() - 86400*2 at 2012-10-25
if (destdir_ctime < 1350983645) or (
(time.time() - destdir_ctime) > maxdelta):
return result
os.remove(srcfile)
action = 'retry:'
if ospe(srcfile) and (ospe(manual_is_not_available_fpath)
or ospe(ospj(destdir, 'manual.html'))):
# we don't try this again
return result
log.msg('%-6s %s/%s at: %s' % (action, extkey, version, url), NL)
log.msg('%-6s %s/%s at: %s' % ('', extkey, version, typo3orgurl),
NL,
dt=' ')
if not ospe(srcfile):
wgetsource = 'http://typo3.org/extension-manuals/%s/%s/sxw/?no_cache=1' % (
extkey, version)
cmd = '/usr/bin/wget -q --output-document=%s %s' % (srcfile.replace(
'/', os.sep), wgetsource)
cmd = 'wget -q --output-document=%s %s' % (srcfile.replace(
'/', os.sep), wgetsource)
devnull = file('/dev/null', 'w')
retCode = subprocess.call(cmd,
shell=True,
stdout=devnull,
stderr=devnull)
devnull.close()
if retCode != 0:
result = 'wget returned error code %s when getting manual.sxw to: %s' % (
retCode, url)
log.msg(result, NL * 2)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
if not ospe(srcfile):
result = 'no manual.sxw found after wget at: %s' % url
log.msg(result, NL * 2)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
tested = False
try:
iszipfile = zipfile.is_zipfile(srcfile)
tested = True
except:
pass
if not tested:
result = "could not test manual.sxw at: %s" % url
log.msg(result, NL * 2)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
if not iszipfile:
result = "%-6s %s/%s: manual.sxw is not a zipfile" % ('bad:', extkey,
version)
log.msg(result, NL * 2, dt=' ')
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
error = ''
try:
zf = zipfile.ZipFile(srcfile)
testresult = zf.testzip()
error = None
except IOError:
error = 'IOError'
except:
error = 'Unknown Error'
if error:
result = "%-6s %s/%s: manual.sxw looks like a zipfile but the zipfiletest gives an '%s'" % (
'bad:', extkey, version, error)
log.msg(result, NL * 2, dt=' ')
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
if testresult:
result = "%-6s %s/%s: manual.sxw looks like a zipfile but is somehow damaged" % (
'bad:', extkey, version)
log.msg(result, NL * 2, dt=' ')
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
srcfile = ospj(destdir, 'manual.sxw')
destfile = ospj(destdir, 'manual.html')
error = convertsxw2html(srcfile, destfile)
if error:
result = '%s at: %s' % (error, url)
log.msg(result, NL * 2)
f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w')
f2.write(result)
f2.close()
return result
cmd = 'chmod +r ' + ospj(destdir, '*')
subprocess.call(cmd, shell=True)
srcfile = ospj(destdir, 'manual.html')
destfile = ospj(destdir, 'manual-cleaned.html')
if ospe(srcfile):
if ospe(manualIsNotAvailableFile):
os.remove(manualIsNotAvailableFile)
copyclean.main(srcfile, destfile)
# manual-cleaned.html -> manual-from-tidy.html
# # step: Use tidy to convert from HTML-4 to XHTML
# tidy -asxhtml -utf8 -f $EXTENSIONS/$EXTKEY/nightly/tidy-error-log.txt -o $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/1-cleaned.html
srcfile = ospj(destdir, 'manual-cleaned.html')
destfile = ospj(destdir, 'manual-from-tidy.html')
errorfile = ospj(destdir, 'tidy-error-log.txt')
cmd = ' '.join([
'tidy', '-asxhtml', '-utf8', '-f', errorfile, '-o', destfile, srcfile
])
if os.path.exists(destfile):
os.remove(destfile)
returncode = subprocess.call(cmd, shell=True)
if not ospe(destfile):
result = 'could not create %s at %s' % (destfile, url)
log.msg(result, NL * 2)
return result
# manual-from-tidy.html -> manual.rst
# python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt
srcfile = ospj(destdir, 'manual-from-tidy.html')
destfile = ospj(destdir, 'manual.rst')
treefile = f3name = ospj(destdir, 'restparser-tree.txt')
logfile = f4name = ospj(destdir, 'restparser-log.txt')
treefile = f3name = None
logfile = f4name = None
tablesas = 'flt'
if ospe(destfile):
os.remove(destfile)
# ooxhtml2rst.main(f1name, f2name, f3name=None, f4name=None, appendlog=0, taginfo=0, tablesas='dl'):
ooxhtml2rst.main(srcfile,
destfile,
treefile,
logfile,
appendlog=0,
taginfo=0,
tablesas=tablesas)
if not ospe(destfile):
result = 'could not create %s at %s' % (destfile, url)
log.msg(result, NL * 2)
return result
if 1 and 'experimental':
# manual-from-tidy.html -> manual-for-sphinx.rst
# python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt
srcfile = ospj(destdir, 'manual-from-tidy.html')
destfile = ospj(destdir, 'manual-for-sphinx.rst')
treefile = f3name = None
logfile = f4name = None
tablesas = 'dl'
if ospe(destfile):
os.remove(destfile)
ooxhtml2rst.main(srcfile, destfile, treefile, logfile)
if not ospe(destfile):
result = 'could not create %s at %s' % (destfile, url)
log.msg(result, NL * 2)
return result
# step: Generate single HTML file from single reST file using the Docutils (no Sphinx involved here)
# rst2htmltypo3.py --link-stylesheet --stylesheet-path=/css/typo3_docutils_styles.css --warnings=$EXTENSIONS/$EXTKEY/nightly/rst2html-warnings.txt $EXTENSIONS/$EXTKEY/nightly/manual.rst $EXTENSIONS/$EXTKEY/nightly/index.html
srcfile = ospj(destdir, 'manual.rst')
destfile = ospj(destdir, 'index.html')
warningsfile = ospj(destdir, 'rst2html-warnings.txt')
if os.path.exists(destfile):
os.remove(destfile)
cmd = ' '.join([
usr_bin_python,
rst2html_script,
'--source-link',
# '--link-stylesheet',
# '--stylesheet-path=%s' % stylesheet_path,
'--template=%s' % absPathToTemplate,
'--field-name-limit=0',
'--warnings=%s' % warningsfile,
srcfile,
destfile,
])
devnull = file('/dev/null', 'w')
#returncode = subprocess.call(cmd, stdout=devnull, stderr=devnull, shell=True)
returncode = subprocess.call(cmd, shell=True)
devnull.close()
if not ospe(destfile):
result = 'could not create index.html from manual.rst at %s' % url
log.msg(result, NL)
return result
else:
log.msg('Ok!', NL, cnt=True)
if 1 and 'experimental':
# step: Generate single HTML file from single reST file using the Docutils (no Sphinx involved here)
# rst2htmltypo3.py --link-stylesheet --stylesheet-path=/css/typo3_docutils_styles.css --warnings=$EXTENSIONS/$EXTKEY/nightly/rst2html-warnings.txt $EXTENSIONS/$EXTKEY/nightly/manual.rst $EXTENSIONS/$EXTKEY/nightly/index.html
srcfile = ospj(destdir, 'manual-for-sphinx.rst')
destfile = ospj(destdir, 'index2.html')
warningsfile = ospj(destdir, 'rst2html-warnings2.txt')
if os.path.exists(destfile):
os.remove(destfile)
cmd = ' '.join([
usr_bin_python,
rst2html_script,
'--source-link',
# '--link-stylesheet',
# '--stylesheet-path=%s' % stylesheet_path,
'--template=%s' % absPathToTemplate,
'--field-name-limit=0',
'--warnings=%s' % warningsfile,
srcfile,
destfile,
])
devnull = file('/dev/null', 'w')
#returncode = subprocess.call(cmd, stdout=devnull, stderr=devnull, shell=True)
returncode = subprocess.call(cmd, shell=True)
devnull.close()
if not ospe(destfile):
result = 'could not create index2.html from manual-for-sphinx.rst at %s' % url
log.msg(result, NL * 2)
return result
else:
log.msg('Ok!', NL * 2)