def _build_html_for_file_ids(tree, start, end): """Write HTML files for file IDs from ``start`` to ``end``. Return None if all goes well, a tuple of (stringified exception, exc type, exc value, file ID, file path) if something goes wrong while htmlifying a file. This is the top-level function of an HTML worker process. Log progress to a file named "build-html-<start>-<end>.log". """ path = '(no file yet)' id = -1 try: # We might as well have this write its log directly rather than returning # them to the master process, since it's already writing the built HTML # directly, since that probably yields better parallelism. conn = connect_db(tree.target_folder) # TODO: Replace this ad hoc logging with the logging module (or something # more humane) so we can get some automatic timestamps. If we get # timestamps spit out in the parent process, we don't need any of the # timing or counting code here. with open_log(tree, 'build-html-%s-%s.log' % (start, end)) as log: # Load htmlifier plugins: plugins = load_htmlifiers(tree) for plugin in plugins: plugin.load(tree, conn) start_time = datetime.now() # Fetch and htmlify each document: for num_files, (id, path, icon, text) in enumerate( conn.execute(""" SELECT files.id, path, icon, trg_index.text FROM trg_index, files WHERE trg_index.id = files.id AND trg_index.id >= ? AND trg_index.id <= ? """, [start, end]), 1): dst_path = os.path.join(tree.target_folder, path + '.html') log.write('Starting %s.\n' % path) htmlify(tree, conn, icon, path, text, dst_path, plugins) conn.commit() conn.close() # Write time information: time = datetime.now() - start_time log.write('Finished %s files in %s.\n' % (num_files, time)) except Exception as exc: type, value, traceback = exc_info() return format_exc(), type, value, id, path
def _build_html_for_file_ids(tree, start, end): """Write HTML files for file IDs from ``start`` to ``end``. Return None if all goes well, a tuple of (stringified exception, exc type, exc value, file ID, file path) if something goes wrong while htmlifying a file. This is the top-level function of an HTML worker process. Log progress to a file named "build-html-<start>-<end>.log". """ path = '(no file yet)' id = -1 try: # We might as well have this write its log directly rather than returning # them to the master process, since it's already writing the built HTML # directly, since that probably yields better parallelism. conn = connect_db(tree.target_folder) # TODO: Replace this ad hoc logging with the logging module (or something # more humane) so we can get some automatic timestamps. If we get # timestamps spit out in the parent process, we don't need any of the # timing or counting code here. with open_log(tree, 'build-html-%s-%s.log' % (start, end)) as log: # Load htmlifier plugins: plugins = load_htmlifiers(tree) for plugin in plugins: plugin.load(tree, conn) start_time = datetime.now() # Fetch and htmlify each document: for num_files, (id, path, icon, text) in enumerate( conn.execute( """ SELECT files.id, path, icon, trg_index.text FROM trg_index, files WHERE trg_index.id = files.id AND trg_index.id >= ? AND trg_index.id <= ? """, [start, end]), 1): dst_path = os.path.join(tree.target_folder, path + '.html') log.write('Starting %s.\n' % path) htmlify(tree, conn, icon, path, text, dst_path, plugins) conn.commit() conn.close() # Write time information: time = datetime.now() - start_time log.write('Finished %s files in %s.\n' % (num_files, time)) except Exception as exc: type, value, traceback = exc_info() return format_exc(), type, value, id, path