def main(sxwfile): if not sxwfile.endswith('manual.sxw'): return left, right = os.path.split(sxwfile) if left: left, right = os.path.split(left) if not right == '_genesis': return if not left: return left, right = os.path.split(left) finalsourcedir = left.replace('\\', '/') finaldocumentationdir = finalsourcedir repositoryname = os.path.split( os.path.split(finaldocumentationdir)[0])[1].replace('\\', '/') finalbuilddir = ospj(left, '_make', 'build').replace('\\', '/') finalnotversioneddir = ospj(left, '_not_versioned').replace('\\', '/') sxwfile = sxwfile.replace('\\', '/') destdir = '/'.join(sxwfile.split('/')[:-1]).replace('\\', '/') print if 1 or printfilenames: print 'repositoryname :', repositoryname print 'finalsourcedir :', finalsourcedir print 'finalbuilddir :', finalbuilddir print 'finalnotversioneddir:', finalnotversioneddir print 'destdir :', destdir if not mockup_uno: # sxw -> html srcfile = sxwfile destfile = sxwfile[:-4] + '.html' if 1 and printfilenames: print 1, srcfile print 2, destfile else: print repositoryname, '/'.join(srcfile.split('/')[-2:]) print repositoryname, '/'.join(destfile.split('/')[-2:]) if 1: error = convertsxw2html(srcfile, destfile) if error: print error # html -> cleaned srcfile = sxwfile[:-4] + '.html' destfile = sxwfile[:-4] + '-cleaned.html' if 1 and printfilenames: print 1, srcfile print 2, destfile else: print repositoryname, '/'.join(destfile.split('/')[-2:]) if 1: copyclean.main(srcfile, destfile) # cleaned -> from-tidy srcfile = sxwfile[:-4] + '-cleaned.html' destfile = sxwfile[:-4] + '-from-tidy.html' if 1 and printfilenames: print 1, srcfile print 2, destfile else: print repositoryname, '/'.join(destfile.split('/')[-2:]) if 1: errorfile = ospj(destdir, 'tidy-error-log.txt') cmd = ' '.join([ 'tidy', '-asxhtml', '-utf8', '-f', errorfile, '-o', destfile, srcfile ]) if os.path.exists(destfile): os.remove(destfile) returncode = subprocess.call(cmd, shell=True) # manual-from-tidy.html -> manual.rst # python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt srcfile = ospj(destdir, 'manual-from-tidy.html').replace('\\', '/') destfile = ospj(destdir, 'manual.rst').replace('\\', '/') treefile = f3name = ospj(destdir, 'restparser-tree.txt') logfile = f4name = ospj(destdir, 'restparser-log.txt') if 1 and printfilenames: print 1, srcfile print 2, destfile else: print repositoryname, '/'.join(destfile.split('/')[-2:]) if 1: if ospe(destfile): os.remove(destfile) # ooxhtml2rst.main(f1name, f2name, f3name=None, f4name=None, appendlog=0, taginfo=0): ooxhtml2rst.main(srcfile, destfile, treefile, logfile) # normalize_empty_lines srcfile = ospj(destdir, 'manual.rst').replace('\\', '/') destfile = ospj(destdir, 'manual-empty-lines-normalized.rst').replace('\\', '/') if 1 and printfilenames: print 1, srcfile print 2, destfile else: print repositoryname, '/'.join(destfile.split('/')[-2:]) if 1: normalize_empty_lines.main(srcfile, destfile, 2) os.remove(srcfile) os.rename(destfile, srcfile) # first slicing srcfile = ospj(destdir, 'manual.rst').replace('\\', '/') tempdir = ospj(destdir, 'temp').replace('\\', '/') if 1 and printfilenames: print 1, srcfile print 2, tempdir else: print repositoryname, '/'.join(tempdir.split('/')[-2:]) if 1: slice_to_numbered_files.main(srcfile, tempdir) # final file structure print repositoryname, '/'.join(finalsourcedir.split('/')[-2:]) srcdirimages = destdir if 1 and printfilenames: print 1, tempdir print 2, finalsourcedir print 3, srcdirimages if 1: write_sphinx_structure.main(tempdir, finalsourcedir, srcdirimages, verbose=0) # res files destdir = ospj(finalsourcedir, '_make') if not ospe(destdir): os.makedirs(destdir) files = os.listdir(resdir) for afile in files: if afile in [ 'default_conf.py', 'default_make.bat', 'default_make-html.bat', 'default_Makefile', ]: destname = afile[len('default_'):] destpath = ospj(destdir, destname) if not ospe(destpath): srcpath = ospj(resdir, afile) shutil.copyfile(srcpath, destpath) if not ospe(finalbuilddir): os.makedirs(finalbuilddir) # .gitignore srcpath = ospj(resdir, 'default_gitignore_toplevel') destpath = ospj(finaldocumentationdir, '.gitignore') if not ospe(destpath): shutil.copyfile(srcpath, destpath) # .gitignore srcpath = ospj(resdir, 'default_gitignore_all_in_this_folder') destpath = ospj(finalbuilddir, '.gitignore') if not ospe(destpath): shutil.copyfile(srcpath, destpath) # .gitignore srcpath = ospj(resdir, 'default_gitignore_all_in_this_folder') destpath = ospj(finalnotversioneddir, '.gitignore') if not ospe(destpath): shutil.copyfile(srcpath, destpath) # more # place warnings.txt there for illustration f2path = ospj(finalnotversioneddir, 'warnings.txt') if not ospe(f2path): f2 = file(f2path, 'w') f2.close() print repositoryname, 'done.' return
def work(self): retCode = 0 msg = '' if self.args.outfilename: if self.f2name != self.args.outfilename: return 1, "Specify outfilename without path. Found: '%s'" % self.args.outfilename if self.f2ext != '.html': return 1, "Specify 'outfilename' with '.html'. Found: '%s'" % self.args.outfilename try: f1 = file(self.args.infile) except: retCode, msg = 1, "Cannot read file '%s'" % self.args.infile return retCode, msg if not os.path.isdir(self.args.tempdir): retCode, msg = (1, "Cannot find tempdir '%s'" % self.args.tempdir) return retCode, msg if os.path.isdir(self.safetempdir): try: shutil.rmtree(self.safetempdir) except: retCode, msg = (1, "Cannot remove safetempdir '%s'" % self.safetempdir) return retCode, msg if os.path.isdir(self.safetempdir): retCode, msg = (1, "Cannot remove safetempdir '%s'" % self.safetempdir) return retCode, msg else: try: os.makedirs(self.safetempdir) except: retCode, msg = (1, "Cannot create safetempdir '%s'" % self.safetempdir) return retCode, msg if self.logdir == ospj(self.safetempdir, 'logs'): os.makedirs(self.logdir) else: if not os.path.isdir(self.logdir): return 1, "Cannot find logdir '%s'" % self.logdir try: f2path = ospj(self.logdir, 'test.txt') f2 = file(f2path, 'w') f2.close() os.unlink(f2path) except IOError: return 1, "Cannot write to file in logdir: '%s'" % f2path # let's check the manual.sxw is not corrupted in some way tested = False try: iszipfile = zipfile.is_zipfile(self.f1path) tested = True except: pass if not tested: retCode, msg = (1, "Cannot test '%s' for being a zipfile" % self.f1path) return retCode, msg elif not iszipfile: retCode, msg = (1, "'%s' is not a zipfile" % self.f1path) return retCode, msg else: pass error = None try: zf = zipfile.ZipFile(self.f1path) testresult = zf.testzip() error = None except IOError: error = 'IOError' except: error = 'Unknown Error' if error: retCode, msg = (1, "'%s' looks like a zipfile but could not " "be tested with zipfiletest" % self.f1path) return retCode, msg if testresult: retCode, msg = (1, "'%s' is a zipfile but not legal in some " "aspects" % self.f1path) retCode, msg = convertsxw2html(self.f1path, self.f2path) if retCode: return retCode, msg if 0: cmd = 'chmod +r ' + ospj(self.safetempdir, '*') subprocess.call(cmd, shell=True) if 1 and "convert *.gif to *.gif.png": L = [] dirname = os.path.dirname(self.f2path) for fname in os.listdir(dirname): if fname.lower().startswith( 'manual_html_') and fname.lower().endswith('.gif'): L.append(fname) if L: for fname in L: gifFile = ospj(dirname, fname) im = PIL.Image.open(gifFile) pngFile = gifFile + '.png' im.save(pngFile) f1 = file(self.f2path) data = f1.read() f1.close() for fname in L: data = data.replace(fname, fname + '.png') f2 = file(self.f2path, "w") f2.write(data) f2.close() if 1: try: copyclean.main(self.f2path, self.f2path_cleaned) except: retCode, msg = (1, "Cannot run 'copyclean.main()") return retCode, msg if 1: # manual-cleaned.html -> manual-from-tidy.html # # step: Use tidy to convert from HTML-4 to XHTML # tidy -asxhtml -utf8 -f $EXTENSIONS/$EXTKEY/nightly/tidy-error-log.txt -o $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/1-cleaned.html cmd = ' '.join([ 'tidy', '-asxhtml', '-utf8', '-f', self.f2path_tidy_error_log, '-o', self.f2path_from_tidy, self.f2path_cleaned ]) retCode = subprocess.call(cmd, shell=True) if not ospe(self.f2path_from_tidy): retCode, msg = (1, "Cannot create '%s'" % self.f2path_from_tidy) if 1: # parse from *.html to *.rst arg = Namespace() arg.infile = self.f2path_from_tidy arg.outfile = self.f2path_rst arg.treefile = None # no treefile arg.treefile = self.f2path_rst_treefile arg.logfile = None # no logfile arg.logfile = self.f2path_rst_logfile arg.appendlog = 0 arg.taginfo = 0 tabletypes = ['t3flt', 'dl', 'flt'] tabletypes = ['t3flt', 'dl'] for i, tablesas in enumerate(tabletypes): outfile = arg.outfile[:-3] + tablesas + '.rst' ooxhtml2rst.main(arg.infile, outfile, arg.treefile, arg.logfile, arg.appendlog, arg.taginfo, tablesas) if not os.path.exists(outfile): return 1, "ooxhtml2rst.main(): could not create ReST file '%s'" % outfile if i == 0: self.f2path_rst = outfile arg.treefile = None arg.logfile = None self.rstfilepaths = [] for fname in os.listdir(self.safetempdir): if fname.endswith('.rst'): fpath = ospj(self.safetempdir, fname) if os.path.isfile(fpath): self.rstfilepaths.append(fpath) if 1: # for each of our newly created *.rst provide a Docutils rendering # errorfilename = 'sxw2html-conversion-error.txt' # self.t3docutils_stylesheet_path # self.usr_bin_python # self.t3rst2html_script # self.safetempdir for f2path_rst in self.rstfilepaths: normalize_empty_lines.main(f2path_rst, self.f2path_rst_temp, 2) os.remove(f2path_rst) os.rename(self.f2path_rst_temp, f2path_rst) if 1: arg = Namespace() arg.pathToTemplate = self.t3docutils_template_path for arg.srcfile in self.rstfilepaths: arg.basename = os.path.basename(arg.srcfile) temp = arg.basename temp = temp.replace('.dl.html', '.html') temp = temp.replace('.flt.html', '.html') temp = temp.replace('.t3flt.html', '.html') arg.basename_generic = temp arg.destfile = arg.srcfile[:-4] + '.html' arg.warningsfile = ospj( self.logdir, arg.basename_generic + '.t3rst2html-warnings.txt') cmd = ' '.join([ self.usr_bin_python, self.t3rst2html_script, '--source-link', # '--link-stylesheet', # '--stylesheet-path=%s' % stylesheet_path, '--template=%s' % arg.pathToTemplate, '--field-name-limit=0', '--warnings=%s' % arg.warningsfile, arg.srcfile, arg.destfile, ]) arg.stdout_log = ospj( self.logdir, arg.basename_generic + '.t3rst2html-stdout.txt') arg.stderr_log = ospj( self.logdir, arg.basename_generic + '.t3rst2html-stderr-log.txt') # errors = 'strict', 'replace', 'ignore', 'xmlcharrefreplace', 'backslashreplace' f2stdout = codecs.open(arg.stdout_log, 'w', 'utf-8', errors='backslashreplace') f2stderr = codecs.open(arg.stderr_log, 'w', 'utf-8', errors='backslashreplace') # devnull = file('/dev/null', 'w') returncode = subprocess.call(cmd, stdout=f2stdout, stderr=f2stderr, shell=True) f2stderr.close() f2stdout.close() # returncode = subprocess.call(cmd, shell=True) # devnull.close() # ================================================== # make ./Documentation structure # -------------------------------------------------- if 1 and 'select the rst file with the desired table handling': for self.f2path_rst in self.rstfilepaths: if self.f2path_rst.endswith('.t3flt.rst'): pass if self.f2path_rst.endswith('.dl.rst'): break else: self.f2path_rst = None arg = Namespace() arg.f2path_rst = self.f2path_rst arg.safetempdir_sliced = self.safetempdir_sliced arg.f2path_documentation = self.f2path_documentation arg.srcdirimages = self.safetempdir arg.resdir = self.resdir retCode, msg = write_sphinx_structure.main(arg) # travel all *.rst files in ./Documentation and # insert an intersphinx label before each section # only underlined sections are recognized prepend_sections_with_labels.main(self.f2path_documentation) ## #### ## Ah, oh, uh, this was a silly idea. Forget it. ## ## # travel all *.rst files in ./Documentation and ## # try to check whether the first row of a .. t3-field-list-table:: ## # should be removed. If so do so. ## remove_first_t3fieldlisttable_row.main(self.f2path_documentation) # travel all *.rst files in ./Documentation and # and try to tweak the '.. container:: table-row' of 'definition list tables' tweak_dllisttables.main(self.f2path_documentation) retCode = 0 msg = "Ok" return retCode, msg
def walk_ter_manuals_html_to_rest(rootfolder): prelpath = len(rootfolder) proceeding = True cnt = 0 cntnew = 0 for path, dirs, files in os.walk(rootfolder): proceedwithfile = True destdir = path dirs.sort() if not proceeding: dirs[:] = [] break else: for afile in files: if not proceeding: break if afile == 'manual.html' and len( os.path.split(path)[1].split('.')) == 3: left, version = os.path.split(path) left, extkey = os.path.split(left) manualhtmlfile = os.path.join(path, 'manual.html') manualhtmlfile = os.path.join(path, afile) cleanedfile = os.path.join(destdir, 'manual-cleaned.html') tidyfile = os.path.join(destdir, 'manual-from-tidy.html') rstfile = os.path.join(destdir, 'manual.rst') indexhtmlfile = os.path.join(destdir, 'index.html') finaldestfile = indexhtmlfile cnt += 1 if 0 and 'force rebuild': if os.path.exists(indexhtmlfile): os.remove(indexhtmlfile) if 1: cmd = 'chmod +r ' + os.path.join(destdir, '*') subprocess.call(cmd, shell=True) if proceedwithfile and os.path.exists(finaldestfile): # nothing to do? print '%04d %4d exists: %s' % ( cnt, cntnew, finaldestfile[prelpath:]) proceedwithfile = False if proceedwithfile: # manual.html -> manual-cleaned.html srcfile = manualhtmlfile destfile = cleanedfile print '%04d %4d new: %s' % (cnt, cntnew, srcfile[prelpath:]) print srcfile if os.path.exists(destfile): os.remove(destfile) proceedwithfile = not os.path.exists(destfile) if not proceedwithfile: print 'could not remove %s' % destfile if proceedwithfile: copyclean.main(srcfile, destfile) proceedwithfile = os.path.exists(cleanedfile) if not proceedwithfile: print 'could not create %s' % destfile if proceedwithfile: # manual-cleaned.html -> manual-from-tidy.html # # step: Use tidy to convert from HTML-4 to XHTML # tidy -asxhtml -utf8 -f $EXTENSIONS/$EXTKEY/nightly/tidy-error-log.txt -o $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/1-cleaned.html srdfile = cleanedfile destfile = tidyfile errorfile = os.path.join(destdir, 'tidy-error-log.txt') cmd = ' '.join([ 'tidy', '-asxhtml', '-utf8', '-f', errorfile, '-o', destfile, cleanedfile ]) if os.path.exists(destfile): os.remove(destfile) proceedwithfile = not os.path.exists(destfile) if not proceedwithfile: print 'could not remove %s' % destfile if proceedwithfile: returncode = subprocess.call(cmd, shell=True) proceedwithfile = os.path.exists(destfile) if not proceedwithfile: print 'could not create %s' % destfile if proceedwithfile: # manual-from-tidy.html -> manual.rst # python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt srcfile = tidyfile destfile = rstfile # treefile = f3name = 'restparser-tree.txt' treefile = f3name = None # logfile = f4name = os.path.join(destdir, 'restparser-log.txt') logfile = f4name = None if os.path.exists(destfile): os.remove(destfile) proceedwithfile = not os.path.exists(destfile) if not proceedwithfile: print 'could not remove %s' % destfile if proceedwithfile: # ooxhtml2rst.main(f1name, f2name, f3name=None, f4name=None, appendlog=0, taginfo=0): ooxhtml2rst.main(srcfile, destfile, treefile, logfile) proceedwithfile = os.path.exists(destfile) if not proceedwithfile: print 'could not create %s' % (destfile, ) if proceedwithfile: # step: Generate single HTML file from single reST file using the Docutils (no Sphinx involved here) # rst2htmltypo3.py --link-stylesheet --stylesheet-path=/css/typo3_docutils_styles.css --warnings=$EXTENSIONS/$EXTKEY/nightly/rst2html-warnings.txt $EXTENSIONS/$EXTKEY/nightly/manual.rst $EXTENSIONS/$EXTKEY/nightly/index.html srcfile = os.path.join(destdir, 'manual.rst') destfile = os.path.join(destdir, 'index.html') if proceedwithfile: # manual.rst -> index.html srcfile = rstfile destfile = indexhtmlfile warningsfile = os.path.join(destdir, 'rst2html-warnings.txt') if os.path.exists(destfile): os.remove(destfile) proceedwithfile = not os.path.exists(destfile) if not proceedwithfile: print 'could not remove %s' % destfile if proceedwithfile: cmd = ' '.join([ usr_bin_python, rst2html_script, '--source-link', '--link-stylesheet', '--stylesheet-path=%s' % stylesheet_path, '--field-name-limit=0', '--warnings=%s' % warningsfile, srcfile, destfile, ]) returncode = subprocess.call(cmd, shell=True) proceedwithfile = os.path.exists(destfile) if not proceedwithfile: print 'could not create %s' % (destfile, ) else: cntnew += 1 if 0 and cntnew > 100: proceeding = False break print return proceeding
def extensionProcessorDoAll(extkey, version): global globalCountExtensions globalCountExtensions += 1 result = None destdir = '%s/%s/%s' % (dest_path_to_extensions, extkey, version) url = "%s/%s/%s/" % (url_of_extensions, extkey, version) if not ospe(destdir): os.makedirs(destdir) srcfile = '%s/manual.sxw' % destdir if ospe(srcfile): return result log.msg('new: %s/%s at: %s' % (extkey, version, url), NL) wgetsource = 'http://typo3.org/extension-manuals/%s/%s/sxw/?no_cache=1' % ( extkey, version) cmd = '/usr/bin/wget -q --output-document=%s %s' % (srcfile.replace( '/', os.sep), wgetsource) cmd = 'wget -q --output-document=%s %s' % (srcfile.replace( '/', os.sep), wgetsource) retCode = subprocess.call(cmd, shell=True) if retCode != 0: result = 'wget returned error code %s when getting manual.sxw to: %s' % ( retCode, url) log.msg(result, NL) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result if not ospe(srcfile): result = 'no manual.sxw found after wget at: %s' % url log.msg(result, NL) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result tested = False try: iszipfile = zipfile.is_zipfile(srcfile) tested = True except: pass if not tested: result = "could not test manual.sxw at: %s" % url log.msg(result, NL) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result if not iszipfile: result = "manual.sxw is not a zipfile: %s" % url log.msg(result, NL) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result error = '' try: zf = zipfile.ZipFile(srcfile) testresult = zf.testzip() error = None except IOError: error = 'IOError' except: error = 'Unknown Error' if error: result = "manual.sxw looks like a zipfile but the zipfile test gives an '%s' at: %s" % ( error, url) log.msg(result, NL) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result if testresult: result = "manual.sxw looks like a zipfile but is somehow damaged at: %s" % url log.msg(result, NL) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result srcfile = ospj(destdir, 'manual.sxw') destfile = ospj(destdir, 'manual.html') error = convertsxw2html(srcfile, destfile) if error: result = '%s at: %s' % (error, url) log.msg(result, NL) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result cmd = 'chmod +r ' + ospj(destdir, '*') subprocess.call(cmd, shell=True) srcfile = ospj(destdir, 'manual.html') destfile = ospj(destdir, 'manual-cleaned.html') copyclean.main(srcfile, destfile) # manual-cleaned.html -> manual-from-tidy.html # # step: Use tidy to convert from HTML-4 to XHTML # tidy -asxhtml -utf8 -f $EXTENSIONS/$EXTKEY/nightly/tidy-error-log.txt -o $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/1-cleaned.html srcfile = ospj(destdir, 'manual-cleaned.html') destfile = ospj(destdir, 'manual-from-tidy.html') errorfile = ospj(destdir, 'tidy-error-log.txt') cmd = ' '.join([ 'tidy', '-asxhtml', '-utf8', '-f', errorfile, '-o', destfile, srcfile ]) if os.path.exists(destfile): os.remove(destfile) returncode = subprocess.call(cmd, shell=True) if not ospe(destfile): result = 'could not create %s at %s' % (destfile, url) log.msg(result, NL) return result # manual-from-tidy.html -> manual.rst # python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt srcfile = ospj(destdir, 'manual-from-tidy.html') destfile = ospj(destdir, 'manual.rst') treefile = f3name = ospj(destdir, 'restparser-tree.txt') treefile = f3name = None logfile = f4name = ospj(destdir, 'restparser-log.txt') logfile = f4name = None if ospe(destfile): os.remove(destfile) # ooxhtml2rst.main(f1name, f2name, f3name=None, f4name=None, appendlog=0, taginfo=0): ooxhtml2rst.main(srcfile, destfile, treefile, logfile) if not ospe(destfile): result = 'could not create %s at %s' % (destfile, url) log.msg(result, NL) return result # step: Generate single HTML file from single reST file using the Docutils (no Sphinx involved here) # rst2htmltypo3.py --link-stylesheet --stylesheet-path=/css/typo3_docutils_styles.css --warnings=$EXTENSIONS/$EXTKEY/nightly/rst2html-warnings.txt $EXTENSIONS/$EXTKEY/nightly/manual.rst $EXTENSIONS/$EXTKEY/nightly/index.html srcfile = ospj(destdir, 'manual.rst') destfile = ospj(destdir, 'index.html') warningsfile = ospj(destdir, 'rst2html-warnings.txt') if os.path.exists(destfile): os.remove(destfile) cmd = ' '.join([ usr_bin_python, rst2html_script, '--source-link', # '--link-stylesheet', # '--stylesheet-path=%s' % stylesheet_path, '--template=%s' % absPathToTemplate, '--field-name-limit=0', '--warnings=%s' % warningsfile, srcfile, destfile, ]) devnull = file('/dev/null', 'w') #returncode = subprocess.call(cmd, stdout=devnull, stderr=devnull, shell=True) returncode = subprocess.call(cmd, shell=True) devnull.close() if not ospe(destfile): result = 'could not create index.html from manual.rst at %s' % url log.msg(result, NL) return result return result
def extensionProcessor(extkey, version): global globalCountExtensions globalCountExtensions += 1 result = None destdir = '%s/%s/%s' % (dest_path_to_extensions, extkey, version) url = "%s/%s/%s/" % (url_of_extensions, extkey, version) typo3orgurl = 'http://typo3.org/extensions/repository/view/%s/' % extkey manual_is_not_available_fpath = destdir + '/manual-is-not-available.txt' if not ospe(destdir): os.makedirs(destdir) destdir_ctime = os.stat(destdir).st_ctime srcfile = '%s/manual.sxw' % destdir if 0 and "retry those having 'manual-is-not-available.txt'": flagfile = destdir + '/manual-is-not-available.txt' if ospe(srcfile) and ospe(manual_is_not_available_fpath): os.remove(srcfile) os.remove(manual_is_not_available_fpath) if ospe(srcfile) and ospe(ospj(destdir, 'manual.html')): return result action = 'new:' if ospe(manual_is_not_available_fpath): # retry for a while maxdelta = 86400 * 1.5 if ospe(srcfile): # 1350983645 = time.time() - 86400*2 at 2012-10-25 if (destdir_ctime < 1350983645) or ( (time.time() - destdir_ctime) > maxdelta): return result os.remove(srcfile) action = 'retry:' if ospe(srcfile) and (ospe(manual_is_not_available_fpath) or ospe(ospj(destdir, 'manual.html'))): # we don't try this again return result log.msg('%-6s %s/%s at: %s' % (action, extkey, version, url), NL) log.msg('%-6s %s/%s at: %s' % ('', extkey, version, typo3orgurl), NL, dt=' ') if not ospe(srcfile): wgetsource = 'http://typo3.org/extension-manuals/%s/%s/sxw/?no_cache=1' % ( extkey, version) cmd = '/usr/bin/wget -q --output-document=%s %s' % (srcfile.replace( '/', os.sep), wgetsource) cmd = 'wget -q --output-document=%s %s' % (srcfile.replace( '/', os.sep), wgetsource) devnull = file('/dev/null', 'w') retCode = subprocess.call(cmd, shell=True, stdout=devnull, stderr=devnull) devnull.close() if retCode != 0: result = 'wget returned error code %s when getting manual.sxw to: %s' % ( retCode, url) log.msg(result, NL * 2) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result if not ospe(srcfile): result = 'no manual.sxw found after wget at: %s' % url log.msg(result, NL * 2) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result tested = False try: iszipfile = zipfile.is_zipfile(srcfile) tested = True except: pass if not tested: result = "could not test manual.sxw at: %s" % url log.msg(result, NL * 2) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result if not iszipfile: result = "%-6s %s/%s: manual.sxw is not a zipfile" % ('bad:', extkey, version) log.msg(result, NL * 2, dt=' ') f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result error = '' try: zf = zipfile.ZipFile(srcfile) testresult = zf.testzip() error = None except IOError: error = 'IOError' except: error = 'Unknown Error' if error: result = "%-6s %s/%s: manual.sxw looks like a zipfile but the zipfiletest gives an '%s'" % ( 'bad:', extkey, version, error) log.msg(result, NL * 2, dt=' ') f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result if testresult: result = "%-6s %s/%s: manual.sxw looks like a zipfile but is somehow damaged" % ( 'bad:', extkey, version) log.msg(result, NL * 2, dt=' ') f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result srcfile = ospj(destdir, 'manual.sxw') destfile = ospj(destdir, 'manual.html') error = convertsxw2html(srcfile, destfile) if error: result = '%s at: %s' % (error, url) log.msg(result, NL * 2) f2 = file(ospj(destdir, manualIsNotAvailableFile), 'w') f2.write(result) f2.close() return result cmd = 'chmod +r ' + ospj(destdir, '*') subprocess.call(cmd, shell=True) srcfile = ospj(destdir, 'manual.html') destfile = ospj(destdir, 'manual-cleaned.html') if ospe(srcfile): if ospe(manualIsNotAvailableFile): os.remove(manualIsNotAvailableFile) copyclean.main(srcfile, destfile) # manual-cleaned.html -> manual-from-tidy.html # # step: Use tidy to convert from HTML-4 to XHTML # tidy -asxhtml -utf8 -f $EXTENSIONS/$EXTKEY/nightly/tidy-error-log.txt -o $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/1-cleaned.html srcfile = ospj(destdir, 'manual-cleaned.html') destfile = ospj(destdir, 'manual-from-tidy.html') errorfile = ospj(destdir, 'tidy-error-log.txt') cmd = ' '.join([ 'tidy', '-asxhtml', '-utf8', '-f', errorfile, '-o', destfile, srcfile ]) if os.path.exists(destfile): os.remove(destfile) returncode = subprocess.call(cmd, shell=True) if not ospe(destfile): result = 'could not create %s at %s' % (destfile, url) log.msg(result, NL * 2) return result # manual-from-tidy.html -> manual.rst # python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt srcfile = ospj(destdir, 'manual-from-tidy.html') destfile = ospj(destdir, 'manual.rst') treefile = f3name = ospj(destdir, 'restparser-tree.txt') logfile = f4name = ospj(destdir, 'restparser-log.txt') treefile = f3name = None logfile = f4name = None tablesas = 'flt' if ospe(destfile): os.remove(destfile) # ooxhtml2rst.main(f1name, f2name, f3name=None, f4name=None, appendlog=0, taginfo=0, tablesas='dl'): ooxhtml2rst.main(srcfile, destfile, treefile, logfile, appendlog=0, taginfo=0, tablesas=tablesas) if not ospe(destfile): result = 'could not create %s at %s' % (destfile, url) log.msg(result, NL * 2) return result if 1 and 'experimental': # manual-from-tidy.html -> manual-for-sphinx.rst # python ooxhtml2rst-work-in-progress.py --treefile=$EXTENSIONS/$EXTKEY/nightly/restparser-tree.txt --logfile=$EXTENSIONS/$EXTKEY/nightly/restparser-log.txt $EXTENSIONS/$EXTKEY/nightly/2-from-tidy.html $EXTENSIONS/$EXTKEY/nightly/manual.rst 1>&2 2>$EXTENSIONS/$EXTKEY/nightly/restparser-errors.txt srcfile = ospj(destdir, 'manual-from-tidy.html') destfile = ospj(destdir, 'manual-for-sphinx.rst') treefile = f3name = None logfile = f4name = None tablesas = 'dl' if ospe(destfile): os.remove(destfile) ooxhtml2rst.main(srcfile, destfile, treefile, logfile) if not ospe(destfile): result = 'could not create %s at %s' % (destfile, url) log.msg(result, NL * 2) return result # step: Generate single HTML file from single reST file using the Docutils (no Sphinx involved here) # rst2htmltypo3.py --link-stylesheet --stylesheet-path=/css/typo3_docutils_styles.css --warnings=$EXTENSIONS/$EXTKEY/nightly/rst2html-warnings.txt $EXTENSIONS/$EXTKEY/nightly/manual.rst $EXTENSIONS/$EXTKEY/nightly/index.html srcfile = ospj(destdir, 'manual.rst') destfile = ospj(destdir, 'index.html') warningsfile = ospj(destdir, 'rst2html-warnings.txt') if os.path.exists(destfile): os.remove(destfile) cmd = ' '.join([ usr_bin_python, rst2html_script, '--source-link', # '--link-stylesheet', # '--stylesheet-path=%s' % stylesheet_path, '--template=%s' % absPathToTemplate, '--field-name-limit=0', '--warnings=%s' % warningsfile, srcfile, destfile, ]) devnull = file('/dev/null', 'w') #returncode = subprocess.call(cmd, stdout=devnull, stderr=devnull, shell=True) returncode = subprocess.call(cmd, shell=True) devnull.close() if not ospe(destfile): result = 'could not create index.html from manual.rst at %s' % url log.msg(result, NL) return result else: log.msg('Ok!', NL, cnt=True) if 1 and 'experimental': # step: Generate single HTML file from single reST file using the Docutils (no Sphinx involved here) # rst2htmltypo3.py --link-stylesheet --stylesheet-path=/css/typo3_docutils_styles.css --warnings=$EXTENSIONS/$EXTKEY/nightly/rst2html-warnings.txt $EXTENSIONS/$EXTKEY/nightly/manual.rst $EXTENSIONS/$EXTKEY/nightly/index.html srcfile = ospj(destdir, 'manual-for-sphinx.rst') destfile = ospj(destdir, 'index2.html') warningsfile = ospj(destdir, 'rst2html-warnings2.txt') if os.path.exists(destfile): os.remove(destfile) cmd = ' '.join([ usr_bin_python, rst2html_script, '--source-link', # '--link-stylesheet', # '--stylesheet-path=%s' % stylesheet_path, '--template=%s' % absPathToTemplate, '--field-name-limit=0', '--warnings=%s' % warningsfile, srcfile, destfile, ]) devnull = file('/dev/null', 'w') #returncode = subprocess.call(cmd, stdout=devnull, stderr=devnull, shell=True) returncode = subprocess.call(cmd, shell=True) devnull.close() if not ospe(destfile): result = 'could not create index2.html from manual-for-sphinx.rst at %s' % url log.msg(result, NL * 2) return result else: log.msg('Ok!', NL * 2)