def build_single_page_version(lang, args, cfg): logging.info(f'Building single page version for {lang}') os.environ['SINGLE_PAGE'] = '1' extra = cfg.data['extra'] extra['single_page'] = True with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md: concatenate(lang, args.docs_dir, single_md) with util.temp_dir() as site_temp: with util.temp_dir() as docs_temp: docs_src_lang = os.path.join(args.docs_dir, lang) docs_temp_lang = os.path.join(docs_temp, lang) shutil.copytree(docs_src_lang, docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: if filename != 'single.md' and filename.endswith( '.md'): os.unlink(os.path.join(root, filename)) cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': site_temp, 'extra': extra, 'nav': [{ cfg.data.get('site_name'): 'single.md' }] }) mkdocs_build.build(cfg) if args.version_prefix: single_page_output_path = os.path.join( args.docs_dir, args.docs_output_dir, args.version_prefix, lang, 'single') else: single_page_output_path = os.path.join( args.docs_dir, args.docs_output_dir, lang, 'single') if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) shutil.copytree(os.path.join(site_temp, 'single'), single_page_output_path) if not args.skip_pdf: with util.temp_dir() as test_dir: single_page_pdf = os.path.abspath( os.path.join(single_page_output_path, f'clickhouse_{lang}.pdf')) extra['single_page'] = False cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': test_dir, 'extra': extra, 'nav': [{ cfg.data.get('site_name'): 'single.md' }] }) mkdocs_build.build(cfg) css_in = ' '.join(website.get_css_in(args)) js_in = ' '.join(website.get_js_in(args)) subprocess.check_call( f'cat {css_in} > {test_dir}/css/base.css', shell=True) subprocess.check_call( f'cat {js_in} > {test_dir}/js/base.js', shell=True) port_for_pdf = util.get_free_port() with socketserver.TCPServer( ('', port_for_pdf), http.server.SimpleHTTPRequestHandler) as httpd: logging.info( f"serving for pdf at port {port_for_pdf}") thread = threading.Thread( target=httpd.serve_forever) with util.cd(test_dir): thread.start() create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', '--no-stop-slow-scripts', '--log-level', 'warn', f'http://localhost:{port_for_pdf}/single/', single_page_pdf ] try: if args.save_raw_single_page: shutil.copytree( test_dir, args.save_raw_single_page) logging.info(' '.join(create_pdf_command)) subprocess.check_call( ' '.join(create_pdf_command), shell=True) finally: httpd.shutdown() thread.join(timeout=5.0) if not args.version_prefix: # maybe enable in future test.test_single_page( os.path.join(test_dir, 'single', 'index.html'), lang) logging.info(f'Finished building single page version for {lang}')
def build_single_page_version(lang, args, cfg): logging.info('Building single page version for ' + lang) os.environ['SINGLE_PAGE'] = '1' with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md: concatenate(lang, args.docs_dir, single_md) with util.temp_dir() as site_temp: with util.temp_dir() as docs_temp: docs_src_lang = os.path.join(args.docs_dir, lang) docs_temp_lang = os.path.join(docs_temp, lang) shutil.copytree(docs_src_lang, docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: if filename != 'single.md' and filename.endswith('.md'): os.unlink(os.path.join(root, filename)) cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': site_temp, 'extra': { 'single_page': True }, 'nav': [ {cfg.data.get('site_name'): 'single.md'} ] }) mkdocs_build.build(cfg) if args.version_prefix: single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, args.version_prefix, lang, 'single') else: single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single') if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) shutil.copytree( os.path.join(site_temp, 'single'), single_page_output_path ) if not args.skip_pdf: single_page_index_html = os.path.abspath(os.path.join(single_page_output_path, 'index.html')) single_page_pdf = single_page_index_html.replace('index.html', 'clickhouse_%s.pdf' % lang) create_pdf_command = ['wkhtmltopdf', '--print-media-type', single_page_index_html, single_page_pdf] logging.debug(' '.join(create_pdf_command)) subprocess.check_call(' '.join(create_pdf_command), shell=True) with util.temp_dir() as test_dir: cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': test_dir, 'extra': { 'single_page': False }, 'nav': [ {cfg.data.get('site_name'): 'single.md'} ] }) mkdocs_build.build(cfg) if not args.version_prefix: # maybe enable in future test.test_single_page(os.path.join(test_dir, 'single', 'index.html'), lang) if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page)
def build_single_page_version(lang, args, cfg): logging.info(f'Building single page version for {lang}') os.environ['SINGLE_PAGE'] = '1' extra = cfg.data['extra'] extra['single_page'] = True with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md: concatenate(lang, args.docs_dir, single_md) with util.temp_dir() as site_temp: with util.temp_dir() as docs_temp: docs_src_lang = os.path.join(args.docs_dir, lang) docs_temp_lang = os.path.join(docs_temp, lang) shutil.copytree(docs_src_lang, docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: if filename != 'single.md' and filename.endswith( '.md'): os.unlink(os.path.join(root, filename)) cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': site_temp, 'extra': extra, 'nav': [{ cfg.data.get('site_name'): 'single.md' }] }) mkdocs_build.build(cfg) if args.version_prefix: single_page_output_path = os.path.join( args.docs_dir, args.docs_output_dir, args.version_prefix, lang, 'single') else: single_page_output_path = os.path.join( args.docs_dir, args.docs_output_dir, lang, 'single') if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) shutil.copytree(os.path.join(site_temp, 'single'), single_page_output_path) logging.info(f'Re-building single page for {lang} pdf/test') with util.temp_dir() as test_dir: single_page_pdf = os.path.abspath( os.path.join(single_page_output_path, f'clickhouse_{lang}.pdf')) extra['single_page'] = False cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': test_dir, 'extra': extra, 'nav': [{ cfg.data.get('site_name'): 'single.md' }] }) mkdocs_build.build(cfg) css_in = ' '.join(website.get_css_in(args)) js_in = ' '.join(website.get_js_in(args)) subprocess.check_call( f'cat {css_in} > {test_dir}/css/base.css', shell=True) subprocess.check_call( f'cat {js_in} > {test_dir}/js/base.js', shell=True) if not args.skip_pdf: port_for_pdf = util.get_free_port() httpd = socketserver.TCPServer( ('', port_for_pdf), http.server.SimpleHTTPRequestHandler) logging.info( f"Serving for {lang} pdf at port {port_for_pdf}") process = multiprocessing.Process( target=httpd.serve_forever) with util.cd(test_dir): process.start() create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', '--disable-javascript', # TODO: restore '--log-level', 'warn', f'http://localhost:{port_for_pdf}/single/', single_page_pdf ] try: if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page) logging.info(' '.join(create_pdf_command)) subprocess.check_call( ' '.join(create_pdf_command), shell=True) finally: logging.info( f'Stop serving for {lang} pdf at port {port_for_pdf}' ) process.kill() while True: time.sleep(0.25) try: process.close() break except ValueError: logging.info( f'Waiting for {lang} httpd at port {port_for_pdf} to stop' ) if not args.version_prefix: # maybe enable in future logging.info(f'Running tests for {lang}') test.test_single_page( os.path.join(test_dir, 'single', 'index.html'), lang) logging.info(f'Finished building single page version for {lang}')
def build_single_page_version(lang, args, nav, cfg): logging.info(f'Building single page version for {lang}') os.environ['SINGLE_PAGE'] = '1' extra = cfg.data['extra'] extra['single_page'] = True extra['is_amp'] = False with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md: concatenate(lang, args.docs_dir, single_md, nav) with util.temp_dir() as site_temp: with util.temp_dir() as docs_temp: docs_src_lang = os.path.join(args.docs_dir, lang) docs_temp_lang = os.path.join(docs_temp, lang) shutil.copytree(docs_src_lang, docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: if filename != 'single.md' and filename.endswith('.md'): os.unlink(os.path.join(root, filename)) cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': site_temp, 'extra': extra, 'nav': [ {cfg.data.get('site_name'): 'single.md'} ] }) if not args.test_only: mkdocs.commands.build.build(cfg) single_page_output_path = os.path.join(args.docs_dir, args.docs_output_dir, lang, 'single') if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) shutil.copytree( os.path.join(site_temp, 'single'), single_page_output_path ) single_page_index_html = os.path.join(single_page_output_path, 'index.html') single_page_content_js = os.path.join(single_page_output_path, 'content.js') with open(single_page_index_html, 'r') as f: sp_prefix, sp_js, sp_suffix = f.read().split('<!-- BREAK -->') with open(single_page_index_html, 'w') as f: f.write(sp_prefix) f.write(sp_suffix) with open(single_page_content_js, 'w') as f: if args.minify: import jsmin sp_js = jsmin.jsmin(sp_js) f.write(sp_js) logging.info(f'Re-building single page for {lang} pdf/test') with util.temp_dir() as test_dir: extra['single_page'] = False cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': test_dir, 'extra': extra, 'nav': [ {cfg.data.get('site_name'): 'single.md'} ] }) mkdocs.commands.build.build(cfg) css_in = ' '.join(website.get_css_in(args)) js_in = ' '.join(website.get_js_in(args)) subprocess.check_call(f'cat {css_in} > {test_dir}/css/base.css', shell=True) subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True) if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page) logging.info(f'Running tests for {lang}') test.test_single_page( os.path.join(test_dir, 'single', 'index.html'), lang) if not args.skip_pdf: single_page_index_html = os.path.join(test_dir, 'single', 'index.html') single_page_pdf = os.path.abspath( os.path.join(single_page_output_path, f'clickhouse_{lang}.pdf') ) with open(single_page_index_html, 'r') as f: soup = bs4.BeautifulSoup( f.read(), features='html.parser' ) soup_prefix = f'file://{test_dir}' for img in soup.findAll('img'): if img['src'].startswith('/'): img['src'] = soup_prefix + img['src'] for script in soup.findAll('script'): script_src = script.get('src') if script_src: script['src'] = soup_prefix + script_src.split('?', 1)[0] for link in soup.findAll('link'): link['href'] = soup_prefix + link['href'].split('?', 1)[0] with open(single_page_index_html, 'w') as f: f.write(str(soup)) create_pdf_command = [ 'wkhtmltopdf', '--print-media-type', '--log-level', 'warn', single_page_index_html, single_page_pdf ] logging.info(' '.join(create_pdf_command)) subprocess.check_call(' '.join(create_pdf_command), shell=True) logging.info(f'Finished building single page version for {lang}')
def build_single_page_version(lang, args, nav, cfg): logging.info(f"Building single page version for {lang}") os.environ["SINGLE_PAGE"] = "1" extra = cfg.data["extra"] extra["single_page"] = True extra["is_amp"] = False single_md_path = get_temporary_file_name(lang, args) with open(single_md_path, "w") as single_md: concatenate(lang, args.docs_dir, single_md, nav) with util.temp_dir() as site_temp: with util.temp_dir() as docs_temp: docs_src_lang = os.path.join(args.docs_dir, lang) docs_temp_lang = os.path.join(docs_temp, lang) shutil.copytree(docs_src_lang, docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: if filename != "single.md" and filename.endswith( ".md"): os.unlink(os.path.join(root, filename)) cfg.load_dict({ "docs_dir": docs_temp_lang, "site_dir": site_temp, "extra": extra, "nav": [{ cfg.data.get("site_name"): "single.md" }], }) if not args.test_only: mkdocs.commands.build.build(cfg) single_page_output_path = os.path.join( args.docs_dir, args.docs_output_dir, lang, "single") if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) shutil.copytree(os.path.join(site_temp, "single"), single_page_output_path) single_page_index_html = os.path.join( single_page_output_path, "index.html") single_page_content_js = os.path.join( single_page_output_path, "content.js") with open(single_page_index_html, "r") as f: sp_prefix, sp_js, sp_suffix = f.read().split( "<!-- BREAK -->") with open(single_page_index_html, "w") as f: f.write(sp_prefix) f.write(sp_suffix) with open(single_page_content_js, "w") as f: if args.minify: import jsmin sp_js = jsmin.jsmin(sp_js) f.write(sp_js) logging.info(f"Re-building single page for {lang} pdf/test") with util.temp_dir() as test_dir: extra["single_page"] = False cfg.load_dict({ "docs_dir": docs_temp_lang, "site_dir": test_dir, "extra": extra, "nav": [{ cfg.data.get("site_name"): "single.md" }], }) mkdocs.commands.build.build(cfg) css_in = " ".join(website.get_css_in(args)) js_in = " ".join(website.get_js_in(args)) subprocess.check_call( f"cat {css_in} > {test_dir}/css/base.css", shell=True) subprocess.check_call( f"cat {js_in} > {test_dir}/js/base.js", shell=True) if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page) logging.info(f"Running tests for {lang}") test.test_single_page( os.path.join(test_dir, "single", "index.html"), lang) logging.info(f"Finished building single page version for {lang}") remove_temporary_files(lang, args)
def build_single_page_version(lang, args, nav, cfg): logging.info(f'Building single page version for {lang}') os.environ['SINGLE_PAGE'] = '1' extra = cfg.data['extra'] extra['single_page'] = True extra['is_amp'] = False single_md_path = get_temporary_file_name(lang, args) with open(single_md_path, 'w') as single_md: concatenate(lang, args.docs_dir, single_md, nav) with util.temp_dir() as site_temp: with util.temp_dir() as docs_temp: docs_src_lang = os.path.join(args.docs_dir, lang) docs_temp_lang = os.path.join(docs_temp, lang) shutil.copytree(docs_src_lang, docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: if filename != 'single.md' and filename.endswith( '.md'): os.unlink(os.path.join(root, filename)) cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': site_temp, 'extra': extra, 'nav': [{ cfg.data.get('site_name'): 'single.md' }] }) if not args.test_only: mkdocs.commands.build.build(cfg) single_page_output_path = os.path.join( args.docs_dir, args.docs_output_dir, lang, 'single') if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) shutil.copytree(os.path.join(site_temp, 'single'), single_page_output_path) single_page_index_html = os.path.join( single_page_output_path, 'index.html') single_page_content_js = os.path.join( single_page_output_path, 'content.js') with open(single_page_index_html, 'r') as f: sp_prefix, sp_js, sp_suffix = f.read().split( '<!-- BREAK -->') with open(single_page_index_html, 'w') as f: f.write(sp_prefix) f.write(sp_suffix) with open(single_page_content_js, 'w') as f: if args.minify: import jsmin sp_js = jsmin.jsmin(sp_js) f.write(sp_js) logging.info(f'Re-building single page for {lang} pdf/test') with util.temp_dir() as test_dir: extra['single_page'] = False cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': test_dir, 'extra': extra, 'nav': [{ cfg.data.get('site_name'): 'single.md' }] }) mkdocs.commands.build.build(cfg) css_in = ' '.join(website.get_css_in(args)) js_in = ' '.join(website.get_js_in(args)) subprocess.check_call( f'cat {css_in} > {test_dir}/css/base.css', shell=True) subprocess.check_call( f'cat {js_in} > {test_dir}/js/base.js', shell=True) if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page) logging.info(f'Running tests for {lang}') test.test_single_page( os.path.join(test_dir, 'single', 'index.html'), lang) logging.info(f'Finished building single page version for {lang}') remove_temporary_files(lang, args)
def build_single_page_version(lang, args, cfg): logging.info('Building single page version for ' + lang) os.environ['SINGLE_PAGE'] = '1' with autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md: concatenate(lang, args.docs_dir, single_md) with temp_dir() as site_temp: with temp_dir() as docs_temp: docs_temp_lang = os.path.join(docs_temp, lang) shutil.copytree(os.path.join(args.docs_dir, lang), docs_temp_lang) for root, _, filenames in os.walk(docs_temp_lang): for filename in filenames: if filename != 'single.md' and filename.endswith('.md'): os.unlink(os.path.join(root, filename)) cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': site_temp, 'extra': { 'single_page': True }, 'nav': [ {cfg.data.get('site_name'): 'single.md'} ] }) mkdocs_build.build(cfg) single_page_output_path = os.path.join(args.docs_dir, args.output_dir, lang, 'single') if os.path.exists(single_page_output_path): shutil.rmtree(single_page_output_path) shutil.copytree( os.path.join(site_temp, 'single'), single_page_output_path ) if not args.skip_pdf: single_page_index_html = os.path.abspath(os.path.join(single_page_output_path, 'index.html')) single_page_pdf = single_page_index_html.replace('index.html', 'clickhouse_%s.pdf' % lang) create_pdf_command = ['wkhtmltopdf', '--print-media-type', single_page_index_html, single_page_pdf] logging.debug(' '.join(create_pdf_command)) subprocess.check_call(' '.join(create_pdf_command), shell=True) with temp_dir() as test_dir: cfg.load_dict({ 'docs_dir': docs_temp_lang, 'site_dir': test_dir, 'extra': { 'single_page': False }, 'nav': [ {cfg.data.get('site_name'): 'single.md'} ] }) mkdocs_build.build(cfg) test.test_single_page(os.path.join(test_dir, 'single', 'index.html'), lang) if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page)