def create_page_key(self, key, values):
"""Create key page."""
TPL_PAGE_KEY = self.template('PAGE_KEY')
var = {}
# Title
var['title'] = key
# Tab Cloud
cloud = self.create_tagcloud_from_key(key)
# Tab Leader
stats = ""
var['leader'] = []
for value in values:
item = {}
docs = self.srvdtb.get_docs_by_key_value(key, value)
item['count'] = len(docs)
item['vfkey'] = valid_filename(key)
item['vfvalue'] = valid_filename(value)
item['name'] = value
var['leader'].append(item)
var['cloud'] = cloud
html = TPL_PAGE_KEY.render(var=var)
return html
def create_page_key_body(self, key):
"""Create key page."""
source_dir = self.srvapp.get_source_path()
values = self.srvdtb.get_all_values_for_key(key)
num_values = len(values)
TPL_BODY_KEY = self.template('BODY_KEY')
# TAB Cloud
cloud = self.create_tagcloud_from_key(key)
# TAB Stats
leader_row = ""
TPL_LEADER_ROW = self.template('LEADER_ROW')
for value in values:
docs = self.srvdtb.get_docs_by_key_value(key, value)
TPL_VALUE_LINK = self.template('LEADER_ROW_VALUE_LINK')
row = {}
row['vfkey'] = valid_filename(key)
row['vfvalue'] = valid_filename(value)
row['title'] = value
value_link = TPL_VALUE_LINK.render(var=row)
leader = {}
leader['url'] = value_link
leader['count_docs'] = len(docs)
leader_row += TPL_LEADER_ROW.render(var=leader)
body = {}
body['cloud'] = cloud
body['leader'] = leader_row
return TPL_BODY_KEY.render(var=body)
def create_tagcloud_from_key(self, key):
"""Create a tag cloud based on key values."""
dkeyurl = {}
for doc in self.srvdtb.get_documents():
tags = self.srvdtb.get_values(doc, key)
url = os.path.basename(doc)[:-5]
for tag in tags:
try:
urllist = dkeyurl[tag]
surllist = set(urllist)
surllist.add(url)
dkeyurl[tag] = list(surllist)
except KeyError:
surllist = set()
surllist.add(url)
dkeyurl[tag] = list(surllist)
max_frequency = set_max_frequency(dkeyurl)
lwords = []
for word in dkeyurl:
len_word = len(word)
if len_word > 0:
lwords.append(word)
len_words = len(lwords)
if len_words > 0:
lwords.sort(key=lambda y: y.lower())
TPL_WORDCLOUD = self.template('WORDCLOUD')
var = {}
var['items'] = []
# ~ html_items = ''
for word in lwords:
frequency = len(dkeyurl[word])
size = get_font_size(frequency, max_frequency)
url = "%s_%s.html" % (valid_filename(key),
valid_filename(word))
tooltip = "%d documents" % frequency
item = {}
item['url'] = url
item['tooltip'] = tooltip
item['size'] = size
item['word'] = word
var['items'].append(item)
html = TPL_WORDCLOUD.render(var=var)
else:
html = ''
return html
def create_metadata_section(self, doc):
"""Return a html block for displaying core and custom keys."""
try:
doc_path = os.path.join(self.srvapp.get_source_path(), doc)
TPL_METADATA_SECTION = self.template('METADATA_SECTION')
var = {}
var['items'] = []
custom_keys = self.srvdtb.get_custom_keys(doc)
custom_props = []
for key in custom_keys:
try:
values = self.get_html_values_from_key(doc, key)
labels = self.get_labels(values)
custom = {}
custom['vfkey'] = valid_filename(key)
custom['key'] = key
custom['labels'] = labels
var['items'].append(custom)
except Exception as error:
self.log.error("Key[%s]: %s", key, error)
var['timestamp'] = self.srvdtb.get_doc_timestamp(doc)
html = TPL_METADATA_SECTION.render(var=var)
except Exception as error:
msgerror = "%s -> %s" % (doc, error)
self.log.error("\t\t%s", msgerror)
html = ''
raise
return html
def create_page_properties(self):
"""Create properties page"""
TPL_PROPS_PAGE = self.template('PAGE_PROPERTIES')
TPL_KEY_MODAL_BUTTON = self.template('KEY_MODAL_BUTTON')
max_frequency = self.get_maxkv_freq()
all_keys = self.srvdtb.get_all_keys()
custom_buttons = ''
var = {}
var['buttons'] = []
for key in all_keys:
ignored_keys = self.srvdtb.get_ignored_keys()
if key not in ignored_keys:
vbtn = {}
vbtn['content'] = self.create_tagcloud_from_key(key)
values = self.srvdtb.get_all_values_for_key(key)
frequency = len(values)
size = get_font_size(frequency, max_frequency)
proportion = int(math.log((frequency * 100) / max_frequency))
vbtn['key'] = key
vbtn['vfkey'] = valid_filename(key)
vbtn['size'] = size
vbtn['tooltip'] = "%d values" % len(values)
button = TPL_KEY_MODAL_BUTTON.render(
var=vbtn
) # % (valid_filename(key), tooltip, size, key, valid_filename(key), valid_filename(key), key, html)
var['buttons'].append(button)
content = TPL_PROPS_PAGE.render(var=var)
# ~ print(content)
self.distribute('properties', content)
def create_metadata_section(self, doc):
"""Return a html block for displaying metadata (keys and values)."""
try:
TPL_METADATA_SECTION = self.template('METADATA_SECTION')
custom_keys = self.srvdtb.get_custom_keys(doc)
var = {}
var['items'] = []
for key in custom_keys:
ckey = {}
ckey['doc'] = doc
ckey['key'] = key
ckey['vfkey'] = valid_filename(key)
try:
values = self.get_html_values_from_key(doc, key)
ckey['labels'] = self.get_labels(values)
var['items'].append(ckey)
except Exception as error:
self.log.error("[BUILDER] - Key[%s]: %s", key, error)
raise
var['timestamp'] = self.srvdtb.get_doc_timestamp(doc)
html = TPL_METADATA_SECTION.render(var=var)
except Exception as error:
msgerror = "%s -> %s" % (doc, error)
self.log.error("[BUILDER] - %s", msgerror)
html = ''
raise
return html
def create_page_properties(self):
"""Create properties page"""
self.properties = {}
TPL_PROPS_PAGE = self.template('PAGE_PROPERTIES')
TPL_KEY_MODAL_BUTTON = self.template('KEY_MODAL_BUTTON')
max_frequency = self.get_maxkv_freq()
all_keys = self.srvdtb.get_all_keys()
var = {}
custom_buttons = []
for key in all_keys:
ignored_keys = self.srvdtb.get_ignored_keys()
if key not in ignored_keys:
html = self.create_tagcloud_from_key(key)
# ~ self.log.error(html)
values = self.srvdtb.get_all_values_for_key(key)
frequency = len(values)
size = get_font_size(frequency, max_frequency)
proportion = int(math.log((frequency * 100) / max_frequency))
tooltip = "%d values" % len(values)
btn = {}
btn['vfkey'] = valid_filename(key)
btn['tooltip'] = tooltip
btn['size'] = size
btn['key'] = key
btn['content'] = html
button = TPL_KEY_MODAL_BUTTON.render(
var=btn
) # % (, tooltip, size, key, valid_filename(key), valid_filename(key), key, html)
custom_buttons.append(button)
self.properties[key] = button
var['buttons'] = custom_buttons
content = TPL_PROPS_PAGE.render(var=var)
self.distribute('properties', content)
def get_doc_card(self, doc):
"""Get card for a given doc"""
var = {}
TPL_DOC_CARD = self.template('CARD_DOC')
TPL_DOC_CARD_CLASS = self.template('CARD_DOC_CLASS')
LINK = self.template('LINK')
title = self.srvdtb.get_values(doc, 'Title')[0]
var['category'] = self.srvdtb.get_values(doc, 'Category')[0]
var['scope'] = self.srvdtb.get_values(doc, 'Scope')[0]
var['tags'] = []
link = {}
for tag in self.srvdtb.get_values(doc, 'Tag'):
link['class'] = TPL_DOC_CARD_CLASS.render()
link['url'] = "Tag_%s.html" % tag
link['title'] = tag
thistag = LINK.render(var=link)
var['tags'].append(thistag)
var['content'] = ''
link = {}
link['class'] = TPL_DOC_CARD_CLASS.render()
link['url'] = valid_filename(doc).replace('.adoc', '.html')
link['title'] = title
var['title'] = LINK.render(var=link)
# ~ link_title = LINK.render(var=link)
if len(var['category']) > 0 and len(var['scope']) > 0:
cat = {}
cat['class'] = "uk-link-heading uk-text-meta"
cat['url'] = "Category_%s.html" % valid_filename(var['category'])
cat['title'] = var['category']
var['link_category'] = LINK.render(var=cat)
sco = {}
sco['class'] = "uk-link-heading uk-text-meta"
sco['url'] = "Category_%s.html" % valid_filename(var['scope'])
sco['title'] = var['scope']
var['link_scope'] = LINK.render(var=sco)
else:
var['link_category'] = ''
var['link_scope'] = ''
# ~ var['timestamp'] = self.srvdtb.get_doc_timestamp(doc)
# ~ var['fuzzy_date'] = fuzzy_date_from_timestamp(var['timestamp'])
var['tooltip'] = title
DOC_CARD = TPL_DOC_CARD.render(var=var)
# ~ self.log.error(DOC_CARD)
return DOC_CARD
def create_page_etype(self, key, subkey):
TPL_PAGE_ETYPE = self.template('PAGE_ETYPE')
TPL_SECTION_ETYPE = self.template('PAGE_SECTION_EVENT_TYPE')
TPL_SWITCHER_ETYPE = self.template('PAGE_SWITCHER_EVENT_TYPE')
TPL_TAB_CENTER = self.template('TAB_CENTER')
TPL_TAB_ITEM = self.template('TAB_ITEM')
key_list = self.srvdtb.get_all_values_for_key(key.title())
def tab_header(docs):
etypes_set = set()
event_types = self.srvapp.get_theme_property('events')
for doc in docs:
subkey_list = self.srvdtb.get_values(doc, subkey.title())
for subkey_item in subkey_list:
etypes_set.add(subkey_item)
items = ''
for etype in sorted(list(etypes_set)):
item = {}
item['name'] = etype.title()
items += TPL_TAB_ITEM.render(var=item)
tab = {}
tab['content'] = items
header = TPL_TAB_CENTER.render(var=tab)
key_list = sorted(list(etypes_set))
return key_list, header
for key_item in key_list:
docs = self.srvdtb.get_docs_by_key_value(key.title(), key_item)
etypes_set, header = tab_header(docs)
# Registered event types
content_key = ''
for etype_item in etypes_set:
section = {}
items = ''
sect_items = 0
for doc in docs:
subkey_item = self.srvdtb.get_values(doc, subkey.title())
if etype_item in subkey_item:
items += self.get_doc_card(doc)
sect_items += 1
section['count_items'] = sect_items
section['count_docs'] = len(docs)
section['items'] = items
content_key += TPL_SECTION_ETYPE.render(var=section)
page_key = {}
page_key['content'] = content_key
content = TPL_SWITCHER_ETYPE.render(var=page_key)
page = {}
page['title'] = key_item
page['header'] = header
page['content'] = content
html = TPL_PAGE_ETYPE.render(var=page)
basename = valid_filename("%s_%s" % (key.title(), key_item))
self.distribute(basename, html)
def get_doc_card(self, doc):
"""Get card for a given doc"""
var = {}
TPL_DOC_CARD = self.template('CARD_DOC')
TPL_DOC_CARD_CLASS = self.template('CARD_DOC_CLASS')
LINK = self.template('LINK')
title = self.srvdtb.get_values(doc, 'Title')[0]
category = self.srvdtb.get_values(doc, 'Category')[0]
scope = self.srvdtb.get_values(doc, 'Scope')[0]
var['category'] = category
var['scope'] = scope
var['data-title'] = "%s%s%s" % (title, category, scope)
var['content'] = ''
link = {}
link['class'] = TPL_DOC_CARD_CLASS.render()
link['url'] = valid_filename(doc).replace('.adoc', '.html')
link['title'] = title
var['link'] = link
var['link_rendered'] = LINK.render(var=link)
var['title'] = LINK.render(var=link)
var['title_nolink'] = title
# ~ link_title = LINK.render(var=link)
if len(var['category']) > 0 and len(var['scope']) > 0:
cat = {}
cat['class'] = "uk-link-heading uk-text-meta"
cat['url'] = "Category_%s.html" % valid_filename(var['category'])
cat['title'] = var['category']
var['link_category'] = LINK.render(var=cat)
sco = {}
sco['class'] = "uk-link-heading uk-text-meta"
sco['url'] = "Scope_%s.html" % valid_filename(var['scope'])
sco['title'] = var['scope']
var['link_scope'] = LINK.render(var=sco)
else:
var['link_category'] = ''
var['link_scope'] = ''
var['timestamp'] = self.srvdtb.get_doc_timestamp(doc)
var['fuzzy_date'] = fuzzy_date_from_timestamp(var['timestamp'])
var['tooltip'] = title
DOC_CARD = TPL_DOC_CARD.render(var=var)
return DOC_CARD
def get_labels(self, values):
"""C0111: Missing function docstring (missing-docstring)."""
var = {}
label_links = ''
TPL_METADATA_VALUE_LINK = self.template('METADATA_VALUE_LINK')
for page, text in values:
var['link_url'] = valid_filename(page)
var['link_name'] = text
if len(text) != 0:
label_links += TPL_METADATA_VALUE_LINK.render(var=var)
return label_links
def get_doc_event_row(self, doc):
"""Get card for a given doc"""
row = {}
LINK = self.template('LINK')
link = {}
link['class'] = "uk-link-heading uk-text-meta"
# Date
row['timestamp'] = self.srvdtb.get_doc_timestamp(doc)
# Team
team = self.srvdtb.get_values(doc, 'Team')[0]
link['url'] = "Team_%s.html" % valid_filename(team)
link['title'] = team
row['team'] = LINK.render(var=link)
# Title
title = self.srvdtb.get_values(doc, 'Title')[0]
link['url'] = "%s.html" % valid_filename(doc).replace('.adoc', '')
link['title'] = title
row['title'] = LINK.render(var=link)
# Category
category = self.srvdtb.get_values(doc, 'Category')[0]
link['url'] = "Category_%s.html" % valid_filename(category)
link['title'] = category
row['category'] = LINK.render(var=link)
# Scope
scope = self.srvdtb.get_values(doc, 'Scope')[0]
link['url'] = "Scope_%s.html" % valid_filename(scope)
link['title'] = scope
row['scope'] = LINK.render(var=link)
# Status
status = self.srvdtb.get_values(doc, 'Status')[0]
link['url'] = "Status_%s.html" % valid_filename(status)
link['title'] = status
row['status'] = LINK.render(var=link)
return row
def get_doc_card(self, doc):
"""Get card for a given doc"""
var = {}
TPL_DOC_CARD = self.template('CARD_DOC')
TPL_DOC_CARD_CLASS = self.template('CARD_DOC_CLASS')
LINK = self.template('LINK')
name = self.srvdtb.get_values(doc, 'Name')[0]
title = self.srvdtb.get_values(doc, 'Title')[0]
var['category'] = self.srvdtb.get_values(doc, 'Category')[0]
var['scope'] = self.srvdtb.get_values(doc, 'Scope')[0]
var['content'] = ''
link = {}
link['class'] = TPL_DOC_CARD_CLASS.render()
link['url'] = valid_filename(doc).replace('.adoc', '.html')
link['title'] = title
var['title'] = LINK.render(var=link)
# ~ link_title = LINK.render(var=link)
if len(var['category']) > 0 and len(var['scope']) > 0:
cat = {}
cat['class'] = "uk-link-heading uk-text-meta"
cat['url'] = "Category_%s.html" % valid_filename(var['category'])
cat['title'] = var['category']
var['link_category'] = LINK.render(var=cat)
sco = {}
sco['class'] = "uk-link-heading uk-text-meta"
sco['url'] = "Scope_%s.html" % valid_filename(var['scope'])
sco['title'] = var['scope']
var['link_scope'] = LINK.render(var=sco)
else:
var['link_category'] = ''
var['link_scope'] = ''
var['tooltip'] = title
var['description'] = self.solutions[name]['description']
DOC_CARD = TPL_DOC_CARD.render(var=var)
# ~ self.log.error(DOC_CARD)
return DOC_CARD
def initialize(self):
"""Initialize application structure."""
# Status
self.running = False
# Get params from command line
self.parameters = self.app.get_params()
for param, value in self.parameters._get_kwargs():
self.log.debug("[SETUP] - KB4IT param: %s = %s", param, value)
# Initialize directories
self.runtime['dir'] = {}
self.runtime['dir']['tmp'] = tempfile.mkdtemp(prefix=LPATH['TMP'] +
'/')
if self.parameters.TARGET_PATH is None:
self.runtime['dir']['target'] = LPATH['WWW']
else:
self.runtime['dir']['target'] = os.path.realpath(
self.parameters.TARGET_PATH)
self.runtime['dir']['source'] = os.path.realpath(
self.parameters.SOURCE_PATH)
self.runtime['dir']['cache'] = os.path.join(
LPATH['CACHE'], valid_filename(self.runtime['dir']['source']))
if not os.path.exists(self.runtime['dir']['cache']):
os.makedirs(self.runtime['dir']['cache'])
# if SORT attribute is given, use it instead of the OS timestamp
if self.parameters.SORT_ATTRIBUTE is None:
self.runtime['sort_attribute'] = 'Timestamp'
else:
self.runtime['sort_attribute'] = self.parameters.SORT_ATTRIBUTE
self.log.debug("[SETUP] - Using sort attribute: %s",
self.runtime['sort_attribute'])
# Initialize docs structure
self.runtime['docs'] = {}
self.runtime['docs']['count'] = 0
self.runtime['docs']['bag'] = []
self.runtime['docs']['target'] = set()
# Load cache dictionary and initialize the new one
self.kbdict_cur = load_kbdict(self.runtime['dir']['source'])
self.kbdict_new['document'] = {}
self.kbdict_new['metadata'] = {}
# Get services
self.get_services()
# Load theme
self.load_theme()
def build_cardset(self, doclist):
"""Default method to build pages paginated"""
CARD_DOC_FILTER_DATA_TITLE = self.template(
'CARD_DOC_FILTER_DATA_TITLE')
CARDS = ""
for doc in doclist:
title = self.srvdtb.get_values(doc, 'Title')[0]
card = self.get_doc_card(doc)
var = {}
var['data-title'] = valid_filename(title)
var['card_doc'] = card
card_search_filter = CARD_DOC_FILTER_DATA_TITLE.render(var=var)
CARDS += card_search_filter
return CARDS
def create_page_stats(self):
"""Create stats page"""
TPL_PAGE_STATS = self.template('PAGE_STATS')
# ~ TPL_ITEM = self.template('KEY_LEADER_ITEM')
var = {}
var['count_docs'] = self.srvapp.get_numdocs()
keys = self.srvdtb.get_all_keys()
var['count_keys'] = len(keys)
var['leader_items'] = []
for key in keys:
values = self.srvdtb.get_all_values_for_key(key)
item = {}
item['key'] = key
item['vfkey'] = valid_filename(key)
item['count_values'] = len(values)
var['leader_items'].append(item)
stats = TPL_PAGE_STATS.render(var=var)
self.distribute('stats', stats)
def formatday(self, day, weekday):
"""Return a day as a table cell."""
eday = 0 # var for checking if it's a event day
cal_date = (self.month, day
) # create a tuple of the calendar month and day
EVENTCAL_TD_NODAY = self.srvbld.template('EVENTCAL_TD_NODAY')
EVENTCAL_TD_DAY_LINK = self.srvbld.template('EVENTCAL_TD_DAY_LINK')
EVENTCAL_TD_DAY_LINK_TODAY = self.srvbld.template(
'EVENTCAL_TD_DAY_LINK_TODAY')
EVENTCAL_TD_DAY_NOLINK = self.srvbld.template('EVENTCAL_TD_DAY_NOLINK')
EVENTCAL_TD_DAY_NOLINK_TODAY = self.srvbld.template(
'EVENTCAL_TD_DAY_NOLINK_TODAY')
EVENT_PAGE = "events_%4d%02d%02d" % (self.year, self.month, day)
EVENT_PAGE_VALID_FNAME = valid_filename(EVENT_PAGE)
link = {}
link['class'] = self.cssclasses[weekday]
link['vfname'] = EVENT_PAGE_VALID_FNAME
link['day'] = day
HTML = ''
if day == 0:
return EVENTCAL_TD_NODAY.render(var=link) # day outside month
else:
try:
self.events_days[self.year]
if cal_date in self.events_days[self.year]:
eday = day
if day == eday:
if self.year == self.now.year and self.month == self.now.month and day == self.now.day:
return EVENTCAL_TD_DAY_LINK_TODAY.render(var=link)
else:
return EVENTCAL_TD_DAY_LINK.render(var=link)
else:
if self.year == self.now.year and self.month == self.now.month and day == self.now.day:
return EVENTCAL_TD_DAY_NOLINK_TODAY.render(var=link)
else:
return EVENTCAL_TD_DAY_NOLINK.render(var=link)
except Exception as error:
if self.year == self.now.year and self.month == self.now.month and day == self.now.day:
return EVENTCAL_TD_DAY_NOLINK_TODAY.render(var=link)
else:
return EVENTCAL_TD_DAY_NOLINK.render(var=link)
def reset(self):
"""WARNING.
Reset environment given source and target directories.
Delete:
- Source directory
- Target directory
- Temporary directory
- Cache directory
- KB4IT database file for this environment
WARNING!!!
Please, note: if you pass the wrong directory...
"""
self.kbdict_new = {}
self.kbdict_cur = {}
filename = valid_filename(self.get_source_path())
kdbdict = 'kbdict-%s.json' % filename
KB4IT_DB_FILE = os.path.join(LPATH['DB'], kdbdict)
delete_target_contents(self.runtime['dir']['cache'])
self.log.info("[RESET] -DIR[%s] deleted", self.runtime['dir']['cache'])
delete_target_contents(self.runtime['dir']['tmp'])
self.log.info("[RESET] -DIR[%s] deleted", self.runtime['dir']['tmp'])
delete_target_contents(self.get_source_path())
self.log.info("[RESET] -DIR[%s] deleted", self.get_source_path())
delete_target_contents(self.get_target_path())
self.log.info("[RESET] -DIR[%s] deleted", self.get_target_path())
delete_target_contents(KB4IT_DB_FILE)
self.log.info("[RESET] -FILE[%s] deleted", KB4IT_DB_FILE)
self.log.info("KB4IT environment reset")
def stage_03_preprocessing(self):
"""
Extract metadata from source docs into a dict.
Create metadata section for each adoc and insert it after the
EOHMARK.
In this way, after being compiled into HTML, final adocs are
browsable throught its metadata.
"""
self.log.debug("[PREPROCESSING] - Start")
def clean_cache():
missing = []
try:
for docname in self.kbdict_cur['document']:
docpath = os.path.join(self.get_source_path(), docname)
if not os.path.exists(docpath):
missing.append(docname)
except KeyError:
pass # skip
if len(missing) == 0:
self.log.debug("[PREPROCESSING] - Cache is empty")
for docname in missing:
docname = docname.replace('.adoc', '')
self.delete_document(docname)
self.log.debug("[PREPROCESSING] - Clean up cache")
clean_cache()
for source in self.runtime['docs']['bag']:
docname = os.path.basename(source)
# Get metadata
docpath = os.path.join(self.get_source_path(), docname)
keys = get_asciidoctor_attributes(docpath)
# Check if file is valid by checking Title
try:
keys['Title']
except KeyError:
self.runtime['docs']['count'] -= 1
self.log.warning(
"[DOCS] - DOC[%s] doesn't has a title. Skip it.", docname)
continue
self.kbdict_new['document'][docname] = {}
self.log.debug("[PREPROCESSING] - DOC[%s] Preprocessing", docname)
# Add a new document to the database
self.srvdtb.add_document(docname)
# Get datetime timestamp from filesystem and add it as
# attribute
timestamp = file_timestamp(source)
self.srvdtb.add_document_key(docname, 'Timestamp', timestamp)
# Get content
with open(source) as source_adoc:
srcadoc = source_adoc.read()
# To track changes in a document, hashes for metadata and content are created.
# Comparing them with those in the cache, KB4IT determines if a document must be
# compiled again. Very useful to reduce the compilation time.
# Get Document Content and Metadata Hashes
self.kbdict_new['document'][docname][
'content_hash'] = get_hash_from_dict({'content': srcadoc})
self.kbdict_new['document'][docname][
'metadata_hash'] = get_hash_from_dict(keys)
self.kbdict_new['document'][docname]['Timestamp'] = timestamp
# Generate caches
for key in keys:
alist = keys[key]
for value in alist:
if len(value.strip()) == 0:
continue
try:
if key in self.runtime['theme']['date_attributes']:
value = string_timestamp(value)
except KeyError:
pass
if key == 'Tag':
value = value.lower()
self.srvdtb.add_document_key(docname, key, value)
# For each document and for each key/value linked to that document add an entry to kbdic['document']
try:
values = self.kbdict_new['document'][docname][key]
if value not in values:
values.append(value)
self.kbdict_new['document'][docname][key] = sorted(
values)
except KeyError:
self.kbdict_new['document'][docname][key] = [value]
# And viceversa, for each key/value add to kbdict['metadata'] all documents linked
try:
documents = self.kbdict_new['metadata'][key][value]
documents.append(docname)
self.kbdict_new[key][value] = sorted(
documents, key=lambda y: y.lower())
except KeyError:
if key not in self.kbdict_new['metadata']:
self.kbdict_new['metadata'][key] = {}
if value not in self.kbdict_new['metadata'][key]:
self.kbdict_new['metadata'][key][value] = [docname]
# Force compilation (from command line)?
DOC_COMPILATION = False
FORCE_ALL = self.parameters.FORCE
if not FORCE_ALL:
# Get cached document path and check if it exists
cached_document = os.path.join(
self.runtime['dir']['cache'],
docname.replace('.adoc', '.html'))
cached_document_exists = os.path.exists(cached_document)
# Compare the document with the one in the cache
if not cached_document_exists:
DOC_COMPILATION = True
REASON = "Not cached"
else:
try:
hash_new = self.kbdict_new['document'][docname][
'content_hash'] + self.kbdict_new['document'][
docname]['metadata_hash']
hash_cur = self.kbdict_cur['document'][docname][
'content_hash'] + self.kbdict_cur['document'][
docname]['metadata_hash']
DOC_COMPILATION = hash_new != hash_cur
REASON = "Hashes differ? %s" % DOC_COMPILATION
except Exception as warning:
DOC_COMPILATION = True
REASON = warning
else:
REASON = "Forced"
COMPILE = DOC_COMPILATION or FORCE_ALL
# Save compilation status
self.kbdict_new['document'][docname]['compile'] = COMPILE
if COMPILE:
newadoc = srcadoc.replace(EOHMARK, '', 1)
# Write new adoc to temporary dir
target = "%s/%s" % (self.runtime['dir']['tmp'],
valid_filename(docname))
with open(target, 'w') as target_adoc:
target_adoc.write(newadoc)
self.log.debug("[PREPROCESSING] - DOC[%s] Compile? %s. Reason: %s",
docname, COMPILE, REASON)
self.add_target(docname.replace('.adoc', '.html'))
# Save current status for the next run
save_kbdict(self.kbdict_new, self.get_source_path())
# Build a list of documents sorted by timestamp
self.srvdtb.sort_database()
# Documents preprocessing stats
self.log.debug("[PREPROCESSING] - Stats - Documents analyzed: %d",
len(self.runtime['docs']['bag']))
keep_docs = compile_docs = 0
for docname in self.kbdict_new['document']:
if self.kbdict_new['document'][docname]['compile']:
compile_docs += 1
else:
keep_docs += 1
self.log.info("[PREPROCESSING] - Stats - Keep: %d - Compile: %d",
keep_docs, compile_docs)
self.log.debug("[PREPROCESSING] - End")
def stage_04_processing(self):
"""Process all documents."""
self.log.debug("[PROCESSING] - Start")
all_keys = set(self.srvdtb.get_all_keys())
ign_keys = set(self.srvdtb.get_ignored_keys())
available_keys = list(all_keys - ign_keys)
K_PATH = []
KV_PATH = []
for key in sorted(available_keys):
COMPILE_KEY = False
FORCE_ALL = self.parameters.FORCE
values = self.srvdtb.get_all_values_for_key(key)
for value in values:
COMPILE_VALUE = False
key_value_docs_new = self.get_kbdict_value(key,
value,
new=True)
key_value_docs_cur = self.get_kbdict_value(key,
value,
new=False)
if key_value_docs_new != key_value_docs_cur:
COMPILE_VALUE = True
COMPILE_VALUE = COMPILE_VALUE or FORCE_ALL
COMPILE_KEY = COMPILE_KEY or COMPILE_VALUE
KV_PATH.append((key, value, COMPILE_VALUE))
self.log.debug("[PROCESSING] - KEY[%s] VALUE[%s] Compile? %s",
key, value, COMPILE_VALUE)
COMPILE_KEY = COMPILE_KEY or FORCE_ALL
K_PATH.append((key, values, COMPILE_KEY))
self.log.debug("[PROCESSING] - KEY[%s] Compile? %s", key,
COMPILE_KEY)
# To compile or not to compile :)
# # Keys
for kpath in K_PATH:
key, values, COMPILE_KEY = kpath
docname = "%s.adoc" % valid_filename(key)
if COMPILE_KEY:
fpath = os.path.join(self.runtime['dir']['tmp'], docname)
html = self.srvthm.create_page_key(key, values)
with open(fpath, 'w') as fkey:
fkey.write(html)
self.add_target(docname.replace('.adoc', '.html'))
# # Keys/Values
for kvpath in KV_PATH:
key, value, COMPILE_VALUE = kvpath
docs = self.get_kbdict_value(key, value, new=True)
sorted_docs = self.srvdtb.sort_by_date(docs)
basename = "%s_%s" % (valid_filename(key), valid_filename(value))
pagination = {}
pagination['key'] = key
pagination['value'] = value
pagination['basename'] = basename
pagination['doclist'] = sorted_docs
pagination['title'] = None
pagination['function'] = 'build_cardset'
pagination['template'] = 'PAGE_PAGINATION_HEAD'
if COMPILE_VALUE:
pagination['fake'] = False
pagelist = self.srvthm.build_pagination(pagination)
else:
pagination['fake'] = True
pagelist = self.srvthm.build_pagination(pagination)
for page in pagelist:
docname = "%s.html" % page
self.add_target(docname.replace('.adoc', '.html'))
self.log.debug("[PROCESSING] - Finish processing keys")
self.log.debug("[PROCESSING] - Start processing theme")
self.srvthm.build()
self.log.debug("[PROCESSING] - End processing theme")
self.log.info("[PROCESSING] - Target docs: %d",
len(self.runtime['docs']['target']))
self.log.debug("[PROCESSING] - End")
def generate_sources(self):
"""Generate pages from Awesome Selfhosted README.md"""
adoc = ""
if not os.path.exists('src'):
os.makedirs('src')
TMPDIR = self.srvapp.get_temp_path()
AWESOME_README_LOCAL = os.path.join(TMPDIR, 'awesome-selfhosted.md')
self.download(AWESOME_README_INTERNET, AWESOME_README_LOCAL)
with open(AWESOME_README_LOCAL, 'r') as fin:
lines = fin.read().splitlines()
n = 0
for line in lines:
if line.startswith(TAG_START):
s = n + 1
if line.startswith(TAG_END):
e = n
n += 1
for line in lines[s:e]:
# Get topic, category and subcategory
if line.startswith('#'):
header = line[:line.find(' ')]
len_header = len(header)
if len_header == 2:
topic = line[line.find(' ') + 1:]
category = ""
subcategory = ""
if len_header == 3:
category = line[line.find(' ') + 1:]
subcategory = ""
if len_header == 4:
subcategory = line[line.find(' ') + 1:]
# Fix wrong levels
if category == '' and subcategory != '':
category = subcategory
subcategory = ''
if len_header == 2:
adoc += ". <<%s#, %s>>\n" % (valid_filename(
"Topic_%s" % topic), topic)
elif len_header == 3:
adoc += ".. <<%s#, %s>>\n" % (valid_filename(
"Category_%s" % category), category)
elif len_header == 4:
adoc += "... <<%s#, %s>>\n" % (valid_filename(
"Subcategory_%s" % subcategory), subcategory)
# Get awesome selfhosted solutions properties
if line.startswith('- '):
solution = {}
solution['topic'] = topic
solution['category'] = category
solution['subcategory'] = subcategory
# Language
sl = line[:-1].rfind('`')
lang = line[sl + 1:-1]
lang = lang.replace('#', '-SHARP')
lang = lang.replace('+', '-PLUS')
line = line[:sl - 1]
solution['language'] = lang
# License
sl = line[:-1].rfind('`')
lic = line[sl + 1:-1]
line = line[:sl - 1]
solution['license'] = lic
# Demo and Source Code links
ssd = line.rfind('[Demo]')
has_demo = ssd > 0
ssc = line.rfind('[Source Code]')
has_code = ssc > 0
if has_demo or has_code:
if has_demo and has_code:
eol = ssd
links = line[ssd:-1]
eud = links.find(')')
url_demo = links[7:eud]
url_code = links[eud + 17:-1]
else:
if has_demo:
eol = ssd
links = line[ssd:-1]
url_demo = links[7:-1]
url_code = ""
if has_code:
eol = ssc
links = line[ssc:-1]
url_demo = ""
url_code = links[14:-1]
else:
eol = None
url_demo = ""
url_code = ""
solution['url_demo'] = url_demo
solution['url_code'] = url_code
if eol is not None:
line = line[:eol - 1]
# Solution name and url
es = line.find(')')
core = line[:es]
name = core[core.find('[') + 1:core.find(']')].strip()
url = core[len(name) + 5:es]
solution['name'] = name
solution['url'] = url
# Description
line = line[es + 1:]
ed = line.find('(')
if ed < 0:
ed = line.find('`')
description = line[2:ed].strip()
solution['description'] = description
self.solutions[name] = solution
self.write_page(solution)
self.index = adoc
def write_page(self, solution):
PAGE = self.template('PAGE')
NAME = valid_filename(solution['name'])
CONTENT = PAGE.render(var=solution)
self.distribute_to_source(NAME, CONTENT)