def create_page_key(self, key, values): """Create key page.""" TPL_PAGE_KEY = self.template('PAGE_KEY') var = {} # Title var['title'] = key # Tab Cloud cloud = self.create_tagcloud_from_key(key) # Tab Leader stats = "" var['leader'] = [] for value in values: item = {} docs = self.srvdtb.get_docs_by_key_value(key, value) item['count'] = len(docs) item['vfkey'] = valid_filename(key) item['vfvalue'] = valid_filename(value) item['name'] = value var['leader'].append(item) var['cloud'] = cloud html = TPL_PAGE_KEY.render(var=var) return html
def create_page_key_body(self, key): """Create key page.""" source_dir = self.srvapp.get_source_path() values = self.srvdtb.get_all_values_for_key(key) num_values = len(values) TPL_BODY_KEY = self.template('BODY_KEY') # TAB Cloud cloud = self.create_tagcloud_from_key(key) # TAB Stats leader_row = "" TPL_LEADER_ROW = self.template('LEADER_ROW') for value in values: docs = self.srvdtb.get_docs_by_key_value(key, value) TPL_VALUE_LINK = self.template('LEADER_ROW_VALUE_LINK') row = {} row['vfkey'] = valid_filename(key) row['vfvalue'] = valid_filename(value) row['title'] = value value_link = TPL_VALUE_LINK.render(var=row) leader = {} leader['url'] = value_link leader['count_docs'] = len(docs) leader_row += TPL_LEADER_ROW.render(var=leader) body = {} body['cloud'] = cloud body['leader'] = leader_row return TPL_BODY_KEY.render(var=body)
def create_tagcloud_from_key(self, key): """Create a tag cloud based on key values.""" dkeyurl = {} for doc in self.srvdtb.get_documents(): tags = self.srvdtb.get_values(doc, key) url = os.path.basename(doc)[:-5] for tag in tags: try: urllist = dkeyurl[tag] surllist = set(urllist) surllist.add(url) dkeyurl[tag] = list(surllist) except KeyError: surllist = set() surllist.add(url) dkeyurl[tag] = list(surllist) max_frequency = set_max_frequency(dkeyurl) lwords = [] for word in dkeyurl: len_word = len(word) if len_word > 0: lwords.append(word) len_words = len(lwords) if len_words > 0: lwords.sort(key=lambda y: y.lower()) TPL_WORDCLOUD = self.template('WORDCLOUD') var = {} var['items'] = [] # ~ html_items = '' for word in lwords: frequency = len(dkeyurl[word]) size = get_font_size(frequency, max_frequency) url = "%s_%s.html" % (valid_filename(key), valid_filename(word)) tooltip = "%d documents" % frequency item = {} item['url'] = url item['tooltip'] = tooltip item['size'] = size item['word'] = word var['items'].append(item) html = TPL_WORDCLOUD.render(var=var) else: html = '' return html
def create_metadata_section(self, doc): """Return a html block for displaying core and custom keys.""" try: doc_path = os.path.join(self.srvapp.get_source_path(), doc) TPL_METADATA_SECTION = self.template('METADATA_SECTION') var = {} var['items'] = [] custom_keys = self.srvdtb.get_custom_keys(doc) custom_props = [] for key in custom_keys: try: values = self.get_html_values_from_key(doc, key) labels = self.get_labels(values) custom = {} custom['vfkey'] = valid_filename(key) custom['key'] = key custom['labels'] = labels var['items'].append(custom) except Exception as error: self.log.error("Key[%s]: %s", key, error) var['timestamp'] = self.srvdtb.get_doc_timestamp(doc) html = TPL_METADATA_SECTION.render(var=var) except Exception as error: msgerror = "%s -> %s" % (doc, error) self.log.error("\t\t%s", msgerror) html = '' raise return html
def create_page_properties(self): """Create properties page""" TPL_PROPS_PAGE = self.template('PAGE_PROPERTIES') TPL_KEY_MODAL_BUTTON = self.template('KEY_MODAL_BUTTON') max_frequency = self.get_maxkv_freq() all_keys = self.srvdtb.get_all_keys() custom_buttons = '' var = {} var['buttons'] = [] for key in all_keys: ignored_keys = self.srvdtb.get_ignored_keys() if key not in ignored_keys: vbtn = {} vbtn['content'] = self.create_tagcloud_from_key(key) values = self.srvdtb.get_all_values_for_key(key) frequency = len(values) size = get_font_size(frequency, max_frequency) proportion = int(math.log((frequency * 100) / max_frequency)) vbtn['key'] = key vbtn['vfkey'] = valid_filename(key) vbtn['size'] = size vbtn['tooltip'] = "%d values" % len(values) button = TPL_KEY_MODAL_BUTTON.render( var=vbtn ) # % (valid_filename(key), tooltip, size, key, valid_filename(key), valid_filename(key), key, html) var['buttons'].append(button) content = TPL_PROPS_PAGE.render(var=var) # ~ print(content) self.distribute('properties', content)
def create_metadata_section(self, doc): """Return a html block for displaying metadata (keys and values).""" try: TPL_METADATA_SECTION = self.template('METADATA_SECTION') custom_keys = self.srvdtb.get_custom_keys(doc) var = {} var['items'] = [] for key in custom_keys: ckey = {} ckey['doc'] = doc ckey['key'] = key ckey['vfkey'] = valid_filename(key) try: values = self.get_html_values_from_key(doc, key) ckey['labels'] = self.get_labels(values) var['items'].append(ckey) except Exception as error: self.log.error("[BUILDER] - Key[%s]: %s", key, error) raise var['timestamp'] = self.srvdtb.get_doc_timestamp(doc) html = TPL_METADATA_SECTION.render(var=var) except Exception as error: msgerror = "%s -> %s" % (doc, error) self.log.error("[BUILDER] - %s", msgerror) html = '' raise return html
def create_page_properties(self): """Create properties page""" self.properties = {} TPL_PROPS_PAGE = self.template('PAGE_PROPERTIES') TPL_KEY_MODAL_BUTTON = self.template('KEY_MODAL_BUTTON') max_frequency = self.get_maxkv_freq() all_keys = self.srvdtb.get_all_keys() var = {} custom_buttons = [] for key in all_keys: ignored_keys = self.srvdtb.get_ignored_keys() if key not in ignored_keys: html = self.create_tagcloud_from_key(key) # ~ self.log.error(html) values = self.srvdtb.get_all_values_for_key(key) frequency = len(values) size = get_font_size(frequency, max_frequency) proportion = int(math.log((frequency * 100) / max_frequency)) tooltip = "%d values" % len(values) btn = {} btn['vfkey'] = valid_filename(key) btn['tooltip'] = tooltip btn['size'] = size btn['key'] = key btn['content'] = html button = TPL_KEY_MODAL_BUTTON.render( var=btn ) # % (, tooltip, size, key, valid_filename(key), valid_filename(key), key, html) custom_buttons.append(button) self.properties[key] = button var['buttons'] = custom_buttons content = TPL_PROPS_PAGE.render(var=var) self.distribute('properties', content)
def get_doc_card(self, doc): """Get card for a given doc""" var = {} TPL_DOC_CARD = self.template('CARD_DOC') TPL_DOC_CARD_CLASS = self.template('CARD_DOC_CLASS') LINK = self.template('LINK') title = self.srvdtb.get_values(doc, 'Title')[0] var['category'] = self.srvdtb.get_values(doc, 'Category')[0] var['scope'] = self.srvdtb.get_values(doc, 'Scope')[0] var['tags'] = [] link = {} for tag in self.srvdtb.get_values(doc, 'Tag'): link['class'] = TPL_DOC_CARD_CLASS.render() link['url'] = "Tag_%s.html" % tag link['title'] = tag thistag = LINK.render(var=link) var['tags'].append(thistag) var['content'] = '' link = {} link['class'] = TPL_DOC_CARD_CLASS.render() link['url'] = valid_filename(doc).replace('.adoc', '.html') link['title'] = title var['title'] = LINK.render(var=link) # ~ link_title = LINK.render(var=link) if len(var['category']) > 0 and len(var['scope']) > 0: cat = {} cat['class'] = "uk-link-heading uk-text-meta" cat['url'] = "Category_%s.html" % valid_filename(var['category']) cat['title'] = var['category'] var['link_category'] = LINK.render(var=cat) sco = {} sco['class'] = "uk-link-heading uk-text-meta" sco['url'] = "Category_%s.html" % valid_filename(var['scope']) sco['title'] = var['scope'] var['link_scope'] = LINK.render(var=sco) else: var['link_category'] = '' var['link_scope'] = '' # ~ var['timestamp'] = self.srvdtb.get_doc_timestamp(doc) # ~ var['fuzzy_date'] = fuzzy_date_from_timestamp(var['timestamp']) var['tooltip'] = title DOC_CARD = TPL_DOC_CARD.render(var=var) # ~ self.log.error(DOC_CARD) return DOC_CARD
def create_page_etype(self, key, subkey): TPL_PAGE_ETYPE = self.template('PAGE_ETYPE') TPL_SECTION_ETYPE = self.template('PAGE_SECTION_EVENT_TYPE') TPL_SWITCHER_ETYPE = self.template('PAGE_SWITCHER_EVENT_TYPE') TPL_TAB_CENTER = self.template('TAB_CENTER') TPL_TAB_ITEM = self.template('TAB_ITEM') key_list = self.srvdtb.get_all_values_for_key(key.title()) def tab_header(docs): etypes_set = set() event_types = self.srvapp.get_theme_property('events') for doc in docs: subkey_list = self.srvdtb.get_values(doc, subkey.title()) for subkey_item in subkey_list: etypes_set.add(subkey_item) items = '' for etype in sorted(list(etypes_set)): item = {} item['name'] = etype.title() items += TPL_TAB_ITEM.render(var=item) tab = {} tab['content'] = items header = TPL_TAB_CENTER.render(var=tab) key_list = sorted(list(etypes_set)) return key_list, header for key_item in key_list: docs = self.srvdtb.get_docs_by_key_value(key.title(), key_item) etypes_set, header = tab_header(docs) # Registered event types content_key = '' for etype_item in etypes_set: section = {} items = '' sect_items = 0 for doc in docs: subkey_item = self.srvdtb.get_values(doc, subkey.title()) if etype_item in subkey_item: items += self.get_doc_card(doc) sect_items += 1 section['count_items'] = sect_items section['count_docs'] = len(docs) section['items'] = items content_key += TPL_SECTION_ETYPE.render(var=section) page_key = {} page_key['content'] = content_key content = TPL_SWITCHER_ETYPE.render(var=page_key) page = {} page['title'] = key_item page['header'] = header page['content'] = content html = TPL_PAGE_ETYPE.render(var=page) basename = valid_filename("%s_%s" % (key.title(), key_item)) self.distribute(basename, html)
def get_doc_card(self, doc): """Get card for a given doc""" var = {} TPL_DOC_CARD = self.template('CARD_DOC') TPL_DOC_CARD_CLASS = self.template('CARD_DOC_CLASS') LINK = self.template('LINK') title = self.srvdtb.get_values(doc, 'Title')[0] category = self.srvdtb.get_values(doc, 'Category')[0] scope = self.srvdtb.get_values(doc, 'Scope')[0] var['category'] = category var['scope'] = scope var['data-title'] = "%s%s%s" % (title, category, scope) var['content'] = '' link = {} link['class'] = TPL_DOC_CARD_CLASS.render() link['url'] = valid_filename(doc).replace('.adoc', '.html') link['title'] = title var['link'] = link var['link_rendered'] = LINK.render(var=link) var['title'] = LINK.render(var=link) var['title_nolink'] = title # ~ link_title = LINK.render(var=link) if len(var['category']) > 0 and len(var['scope']) > 0: cat = {} cat['class'] = "uk-link-heading uk-text-meta" cat['url'] = "Category_%s.html" % valid_filename(var['category']) cat['title'] = var['category'] var['link_category'] = LINK.render(var=cat) sco = {} sco['class'] = "uk-link-heading uk-text-meta" sco['url'] = "Scope_%s.html" % valid_filename(var['scope']) sco['title'] = var['scope'] var['link_scope'] = LINK.render(var=sco) else: var['link_category'] = '' var['link_scope'] = '' var['timestamp'] = self.srvdtb.get_doc_timestamp(doc) var['fuzzy_date'] = fuzzy_date_from_timestamp(var['timestamp']) var['tooltip'] = title DOC_CARD = TPL_DOC_CARD.render(var=var) return DOC_CARD
def get_labels(self, values): """C0111: Missing function docstring (missing-docstring).""" var = {} label_links = '' TPL_METADATA_VALUE_LINK = self.template('METADATA_VALUE_LINK') for page, text in values: var['link_url'] = valid_filename(page) var['link_name'] = text if len(text) != 0: label_links += TPL_METADATA_VALUE_LINK.render(var=var) return label_links
def get_doc_event_row(self, doc): """Get card for a given doc""" row = {} LINK = self.template('LINK') link = {} link['class'] = "uk-link-heading uk-text-meta" # Date row['timestamp'] = self.srvdtb.get_doc_timestamp(doc) # Team team = self.srvdtb.get_values(doc, 'Team')[0] link['url'] = "Team_%s.html" % valid_filename(team) link['title'] = team row['team'] = LINK.render(var=link) # Title title = self.srvdtb.get_values(doc, 'Title')[0] link['url'] = "%s.html" % valid_filename(doc).replace('.adoc', '') link['title'] = title row['title'] = LINK.render(var=link) # Category category = self.srvdtb.get_values(doc, 'Category')[0] link['url'] = "Category_%s.html" % valid_filename(category) link['title'] = category row['category'] = LINK.render(var=link) # Scope scope = self.srvdtb.get_values(doc, 'Scope')[0] link['url'] = "Scope_%s.html" % valid_filename(scope) link['title'] = scope row['scope'] = LINK.render(var=link) # Status status = self.srvdtb.get_values(doc, 'Status')[0] link['url'] = "Status_%s.html" % valid_filename(status) link['title'] = status row['status'] = LINK.render(var=link) return row
def get_doc_card(self, doc): """Get card for a given doc""" var = {} TPL_DOC_CARD = self.template('CARD_DOC') TPL_DOC_CARD_CLASS = self.template('CARD_DOC_CLASS') LINK = self.template('LINK') name = self.srvdtb.get_values(doc, 'Name')[0] title = self.srvdtb.get_values(doc, 'Title')[0] var['category'] = self.srvdtb.get_values(doc, 'Category')[0] var['scope'] = self.srvdtb.get_values(doc, 'Scope')[0] var['content'] = '' link = {} link['class'] = TPL_DOC_CARD_CLASS.render() link['url'] = valid_filename(doc).replace('.adoc', '.html') link['title'] = title var['title'] = LINK.render(var=link) # ~ link_title = LINK.render(var=link) if len(var['category']) > 0 and len(var['scope']) > 0: cat = {} cat['class'] = "uk-link-heading uk-text-meta" cat['url'] = "Category_%s.html" % valid_filename(var['category']) cat['title'] = var['category'] var['link_category'] = LINK.render(var=cat) sco = {} sco['class'] = "uk-link-heading uk-text-meta" sco['url'] = "Scope_%s.html" % valid_filename(var['scope']) sco['title'] = var['scope'] var['link_scope'] = LINK.render(var=sco) else: var['link_category'] = '' var['link_scope'] = '' var['tooltip'] = title var['description'] = self.solutions[name]['description'] DOC_CARD = TPL_DOC_CARD.render(var=var) # ~ self.log.error(DOC_CARD) return DOC_CARD
def initialize(self): """Initialize application structure.""" # Status self.running = False # Get params from command line self.parameters = self.app.get_params() for param, value in self.parameters._get_kwargs(): self.log.debug("[SETUP] - KB4IT param: %s = %s", param, value) # Initialize directories self.runtime['dir'] = {} self.runtime['dir']['tmp'] = tempfile.mkdtemp(prefix=LPATH['TMP'] + '/') if self.parameters.TARGET_PATH is None: self.runtime['dir']['target'] = LPATH['WWW'] else: self.runtime['dir']['target'] = os.path.realpath( self.parameters.TARGET_PATH) self.runtime['dir']['source'] = os.path.realpath( self.parameters.SOURCE_PATH) self.runtime['dir']['cache'] = os.path.join( LPATH['CACHE'], valid_filename(self.runtime['dir']['source'])) if not os.path.exists(self.runtime['dir']['cache']): os.makedirs(self.runtime['dir']['cache']) # if SORT attribute is given, use it instead of the OS timestamp if self.parameters.SORT_ATTRIBUTE is None: self.runtime['sort_attribute'] = 'Timestamp' else: self.runtime['sort_attribute'] = self.parameters.SORT_ATTRIBUTE self.log.debug("[SETUP] - Using sort attribute: %s", self.runtime['sort_attribute']) # Initialize docs structure self.runtime['docs'] = {} self.runtime['docs']['count'] = 0 self.runtime['docs']['bag'] = [] self.runtime['docs']['target'] = set() # Load cache dictionary and initialize the new one self.kbdict_cur = load_kbdict(self.runtime['dir']['source']) self.kbdict_new['document'] = {} self.kbdict_new['metadata'] = {} # Get services self.get_services() # Load theme self.load_theme()
def build_cardset(self, doclist): """Default method to build pages paginated""" CARD_DOC_FILTER_DATA_TITLE = self.template( 'CARD_DOC_FILTER_DATA_TITLE') CARDS = "" for doc in doclist: title = self.srvdtb.get_values(doc, 'Title')[0] card = self.get_doc_card(doc) var = {} var['data-title'] = valid_filename(title) var['card_doc'] = card card_search_filter = CARD_DOC_FILTER_DATA_TITLE.render(var=var) CARDS += card_search_filter return CARDS
def create_page_stats(self): """Create stats page""" TPL_PAGE_STATS = self.template('PAGE_STATS') # ~ TPL_ITEM = self.template('KEY_LEADER_ITEM') var = {} var['count_docs'] = self.srvapp.get_numdocs() keys = self.srvdtb.get_all_keys() var['count_keys'] = len(keys) var['leader_items'] = [] for key in keys: values = self.srvdtb.get_all_values_for_key(key) item = {} item['key'] = key item['vfkey'] = valid_filename(key) item['count_values'] = len(values) var['leader_items'].append(item) stats = TPL_PAGE_STATS.render(var=var) self.distribute('stats', stats)
def formatday(self, day, weekday): """Return a day as a table cell.""" eday = 0 # var for checking if it's a event day cal_date = (self.month, day ) # create a tuple of the calendar month and day EVENTCAL_TD_NODAY = self.srvbld.template('EVENTCAL_TD_NODAY') EVENTCAL_TD_DAY_LINK = self.srvbld.template('EVENTCAL_TD_DAY_LINK') EVENTCAL_TD_DAY_LINK_TODAY = self.srvbld.template( 'EVENTCAL_TD_DAY_LINK_TODAY') EVENTCAL_TD_DAY_NOLINK = self.srvbld.template('EVENTCAL_TD_DAY_NOLINK') EVENTCAL_TD_DAY_NOLINK_TODAY = self.srvbld.template( 'EVENTCAL_TD_DAY_NOLINK_TODAY') EVENT_PAGE = "events_%4d%02d%02d" % (self.year, self.month, day) EVENT_PAGE_VALID_FNAME = valid_filename(EVENT_PAGE) link = {} link['class'] = self.cssclasses[weekday] link['vfname'] = EVENT_PAGE_VALID_FNAME link['day'] = day HTML = '' if day == 0: return EVENTCAL_TD_NODAY.render(var=link) # day outside month else: try: self.events_days[self.year] if cal_date in self.events_days[self.year]: eday = day if day == eday: if self.year == self.now.year and self.month == self.now.month and day == self.now.day: return EVENTCAL_TD_DAY_LINK_TODAY.render(var=link) else: return EVENTCAL_TD_DAY_LINK.render(var=link) else: if self.year == self.now.year and self.month == self.now.month and day == self.now.day: return EVENTCAL_TD_DAY_NOLINK_TODAY.render(var=link) else: return EVENTCAL_TD_DAY_NOLINK.render(var=link) except Exception as error: if self.year == self.now.year and self.month == self.now.month and day == self.now.day: return EVENTCAL_TD_DAY_NOLINK_TODAY.render(var=link) else: return EVENTCAL_TD_DAY_NOLINK.render(var=link)
def reset(self): """WARNING. Reset environment given source and target directories. Delete: - Source directory - Target directory - Temporary directory - Cache directory - KB4IT database file for this environment WARNING!!! Please, note: if you pass the wrong directory... """ self.kbdict_new = {} self.kbdict_cur = {} filename = valid_filename(self.get_source_path()) kdbdict = 'kbdict-%s.json' % filename KB4IT_DB_FILE = os.path.join(LPATH['DB'], kdbdict) delete_target_contents(self.runtime['dir']['cache']) self.log.info("[RESET] -DIR[%s] deleted", self.runtime['dir']['cache']) delete_target_contents(self.runtime['dir']['tmp']) self.log.info("[RESET] -DIR[%s] deleted", self.runtime['dir']['tmp']) delete_target_contents(self.get_source_path()) self.log.info("[RESET] -DIR[%s] deleted", self.get_source_path()) delete_target_contents(self.get_target_path()) self.log.info("[RESET] -DIR[%s] deleted", self.get_target_path()) delete_target_contents(KB4IT_DB_FILE) self.log.info("[RESET] -FILE[%s] deleted", KB4IT_DB_FILE) self.log.info("KB4IT environment reset")
def stage_03_preprocessing(self): """ Extract metadata from source docs into a dict. Create metadata section for each adoc and insert it after the EOHMARK. In this way, after being compiled into HTML, final adocs are browsable throught its metadata. """ self.log.debug("[PREPROCESSING] - Start") def clean_cache(): missing = [] try: for docname in self.kbdict_cur['document']: docpath = os.path.join(self.get_source_path(), docname) if not os.path.exists(docpath): missing.append(docname) except KeyError: pass # skip if len(missing) == 0: self.log.debug("[PREPROCESSING] - Cache is empty") for docname in missing: docname = docname.replace('.adoc', '') self.delete_document(docname) self.log.debug("[PREPROCESSING] - Clean up cache") clean_cache() for source in self.runtime['docs']['bag']: docname = os.path.basename(source) # Get metadata docpath = os.path.join(self.get_source_path(), docname) keys = get_asciidoctor_attributes(docpath) # Check if file is valid by checking Title try: keys['Title'] except KeyError: self.runtime['docs']['count'] -= 1 self.log.warning( "[DOCS] - DOC[%s] doesn't has a title. Skip it.", docname) continue self.kbdict_new['document'][docname] = {} self.log.debug("[PREPROCESSING] - DOC[%s] Preprocessing", docname) # Add a new document to the database self.srvdtb.add_document(docname) # Get datetime timestamp from filesystem and add it as # attribute timestamp = file_timestamp(source) self.srvdtb.add_document_key(docname, 'Timestamp', timestamp) # Get content with open(source) as source_adoc: srcadoc = source_adoc.read() # To track changes in a document, hashes for metadata and content are created. # Comparing them with those in the cache, KB4IT determines if a document must be # compiled again. Very useful to reduce the compilation time. # Get Document Content and Metadata Hashes self.kbdict_new['document'][docname][ 'content_hash'] = get_hash_from_dict({'content': srcadoc}) self.kbdict_new['document'][docname][ 'metadata_hash'] = get_hash_from_dict(keys) self.kbdict_new['document'][docname]['Timestamp'] = timestamp # Generate caches for key in keys: alist = keys[key] for value in alist: if len(value.strip()) == 0: continue try: if key in self.runtime['theme']['date_attributes']: value = string_timestamp(value) except KeyError: pass if key == 'Tag': value = value.lower() self.srvdtb.add_document_key(docname, key, value) # For each document and for each key/value linked to that document add an entry to kbdic['document'] try: values = self.kbdict_new['document'][docname][key] if value not in values: values.append(value) self.kbdict_new['document'][docname][key] = sorted( values) except KeyError: self.kbdict_new['document'][docname][key] = [value] # And viceversa, for each key/value add to kbdict['metadata'] all documents linked try: documents = self.kbdict_new['metadata'][key][value] documents.append(docname) self.kbdict_new[key][value] = sorted( documents, key=lambda y: y.lower()) except KeyError: if key not in self.kbdict_new['metadata']: self.kbdict_new['metadata'][key] = {} if value not in self.kbdict_new['metadata'][key]: self.kbdict_new['metadata'][key][value] = [docname] # Force compilation (from command line)? DOC_COMPILATION = False FORCE_ALL = self.parameters.FORCE if not FORCE_ALL: # Get cached document path and check if it exists cached_document = os.path.join( self.runtime['dir']['cache'], docname.replace('.adoc', '.html')) cached_document_exists = os.path.exists(cached_document) # Compare the document with the one in the cache if not cached_document_exists: DOC_COMPILATION = True REASON = "Not cached" else: try: hash_new = self.kbdict_new['document'][docname][ 'content_hash'] + self.kbdict_new['document'][ docname]['metadata_hash'] hash_cur = self.kbdict_cur['document'][docname][ 'content_hash'] + self.kbdict_cur['document'][ docname]['metadata_hash'] DOC_COMPILATION = hash_new != hash_cur REASON = "Hashes differ? %s" % DOC_COMPILATION except Exception as warning: DOC_COMPILATION = True REASON = warning else: REASON = "Forced" COMPILE = DOC_COMPILATION or FORCE_ALL # Save compilation status self.kbdict_new['document'][docname]['compile'] = COMPILE if COMPILE: newadoc = srcadoc.replace(EOHMARK, '', 1) # Write new adoc to temporary dir target = "%s/%s" % (self.runtime['dir']['tmp'], valid_filename(docname)) with open(target, 'w') as target_adoc: target_adoc.write(newadoc) self.log.debug("[PREPROCESSING] - DOC[%s] Compile? %s. Reason: %s", docname, COMPILE, REASON) self.add_target(docname.replace('.adoc', '.html')) # Save current status for the next run save_kbdict(self.kbdict_new, self.get_source_path()) # Build a list of documents sorted by timestamp self.srvdtb.sort_database() # Documents preprocessing stats self.log.debug("[PREPROCESSING] - Stats - Documents analyzed: %d", len(self.runtime['docs']['bag'])) keep_docs = compile_docs = 0 for docname in self.kbdict_new['document']: if self.kbdict_new['document'][docname]['compile']: compile_docs += 1 else: keep_docs += 1 self.log.info("[PREPROCESSING] - Stats - Keep: %d - Compile: %d", keep_docs, compile_docs) self.log.debug("[PREPROCESSING] - End")
def stage_04_processing(self): """Process all documents.""" self.log.debug("[PROCESSING] - Start") all_keys = set(self.srvdtb.get_all_keys()) ign_keys = set(self.srvdtb.get_ignored_keys()) available_keys = list(all_keys - ign_keys) K_PATH = [] KV_PATH = [] for key in sorted(available_keys): COMPILE_KEY = False FORCE_ALL = self.parameters.FORCE values = self.srvdtb.get_all_values_for_key(key) for value in values: COMPILE_VALUE = False key_value_docs_new = self.get_kbdict_value(key, value, new=True) key_value_docs_cur = self.get_kbdict_value(key, value, new=False) if key_value_docs_new != key_value_docs_cur: COMPILE_VALUE = True COMPILE_VALUE = COMPILE_VALUE or FORCE_ALL COMPILE_KEY = COMPILE_KEY or COMPILE_VALUE KV_PATH.append((key, value, COMPILE_VALUE)) self.log.debug("[PROCESSING] - KEY[%s] VALUE[%s] Compile? %s", key, value, COMPILE_VALUE) COMPILE_KEY = COMPILE_KEY or FORCE_ALL K_PATH.append((key, values, COMPILE_KEY)) self.log.debug("[PROCESSING] - KEY[%s] Compile? %s", key, COMPILE_KEY) # To compile or not to compile :) # # Keys for kpath in K_PATH: key, values, COMPILE_KEY = kpath docname = "%s.adoc" % valid_filename(key) if COMPILE_KEY: fpath = os.path.join(self.runtime['dir']['tmp'], docname) html = self.srvthm.create_page_key(key, values) with open(fpath, 'w') as fkey: fkey.write(html) self.add_target(docname.replace('.adoc', '.html')) # # Keys/Values for kvpath in KV_PATH: key, value, COMPILE_VALUE = kvpath docs = self.get_kbdict_value(key, value, new=True) sorted_docs = self.srvdtb.sort_by_date(docs) basename = "%s_%s" % (valid_filename(key), valid_filename(value)) pagination = {} pagination['key'] = key pagination['value'] = value pagination['basename'] = basename pagination['doclist'] = sorted_docs pagination['title'] = None pagination['function'] = 'build_cardset' pagination['template'] = 'PAGE_PAGINATION_HEAD' if COMPILE_VALUE: pagination['fake'] = False pagelist = self.srvthm.build_pagination(pagination) else: pagination['fake'] = True pagelist = self.srvthm.build_pagination(pagination) for page in pagelist: docname = "%s.html" % page self.add_target(docname.replace('.adoc', '.html')) self.log.debug("[PROCESSING] - Finish processing keys") self.log.debug("[PROCESSING] - Start processing theme") self.srvthm.build() self.log.debug("[PROCESSING] - End processing theme") self.log.info("[PROCESSING] - Target docs: %d", len(self.runtime['docs']['target'])) self.log.debug("[PROCESSING] - End")
def generate_sources(self): """Generate pages from Awesome Selfhosted README.md""" adoc = "" if not os.path.exists('src'): os.makedirs('src') TMPDIR = self.srvapp.get_temp_path() AWESOME_README_LOCAL = os.path.join(TMPDIR, 'awesome-selfhosted.md') self.download(AWESOME_README_INTERNET, AWESOME_README_LOCAL) with open(AWESOME_README_LOCAL, 'r') as fin: lines = fin.read().splitlines() n = 0 for line in lines: if line.startswith(TAG_START): s = n + 1 if line.startswith(TAG_END): e = n n += 1 for line in lines[s:e]: # Get topic, category and subcategory if line.startswith('#'): header = line[:line.find(' ')] len_header = len(header) if len_header == 2: topic = line[line.find(' ') + 1:] category = "" subcategory = "" if len_header == 3: category = line[line.find(' ') + 1:] subcategory = "" if len_header == 4: subcategory = line[line.find(' ') + 1:] # Fix wrong levels if category == '' and subcategory != '': category = subcategory subcategory = '' if len_header == 2: adoc += ". <<%s#, %s>>\n" % (valid_filename( "Topic_%s" % topic), topic) elif len_header == 3: adoc += ".. <<%s#, %s>>\n" % (valid_filename( "Category_%s" % category), category) elif len_header == 4: adoc += "... <<%s#, %s>>\n" % (valid_filename( "Subcategory_%s" % subcategory), subcategory) # Get awesome selfhosted solutions properties if line.startswith('- '): solution = {} solution['topic'] = topic solution['category'] = category solution['subcategory'] = subcategory # Language sl = line[:-1].rfind('`') lang = line[sl + 1:-1] lang = lang.replace('#', '-SHARP') lang = lang.replace('+', '-PLUS') line = line[:sl - 1] solution['language'] = lang # License sl = line[:-1].rfind('`') lic = line[sl + 1:-1] line = line[:sl - 1] solution['license'] = lic # Demo and Source Code links ssd = line.rfind('[Demo]') has_demo = ssd > 0 ssc = line.rfind('[Source Code]') has_code = ssc > 0 if has_demo or has_code: if has_demo and has_code: eol = ssd links = line[ssd:-1] eud = links.find(')') url_demo = links[7:eud] url_code = links[eud + 17:-1] else: if has_demo: eol = ssd links = line[ssd:-1] url_demo = links[7:-1] url_code = "" if has_code: eol = ssc links = line[ssc:-1] url_demo = "" url_code = links[14:-1] else: eol = None url_demo = "" url_code = "" solution['url_demo'] = url_demo solution['url_code'] = url_code if eol is not None: line = line[:eol - 1] # Solution name and url es = line.find(')') core = line[:es] name = core[core.find('[') + 1:core.find(']')].strip() url = core[len(name) + 5:es] solution['name'] = name solution['url'] = url # Description line = line[es + 1:] ed = line.find('(') if ed < 0: ed = line.find('`') description = line[2:ed].strip() solution['description'] = description self.solutions[name] = solution self.write_page(solution) self.index = adoc
def write_page(self, solution): PAGE = self.template('PAGE') NAME = valid_filename(solution['name']) CONTENT = PAGE.render(var=solution) self.distribute_to_source(NAME, CONTENT)