def header(infolder,
line=comment_LINE,
block=comment_BLOCK,
include=INCLUDE,
exclude=EXCLUDE,
overwrite=False):
# Compile regular expression pattern to match in scanned files
pattern = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)),
*comment_block_comments(block)),
flags=re_DOTALL | re_VERBOSE | re_MULTILINE)
# Define default values
align = _FORMAT['CENTER']
width = 80
# Update values based on INFO file
values = {}
with open(os_path_join(infolder, 'INFO'), 'r', encoding='utf-8') as file:
header = file.read()
match = re_match(r'\s*#\s*format\s+'
r'((?P<align>CENTER|LEFT|RIGHT)\s+)?'
r'(?P<width>\d+)?', header)
if match:
align, width = match.group('align', 'width')
align = _FORMAT.get(align, _FORMAT['CENTER'])
try:
width = int(width)
except TypeError:
pass
# Add leading and trailing empty line
header = '\n{}\n'.format(header[match.end():].strip())
# Get file contents of special files
for filename in _FILES:
try:
with open(os_path_join(infolder, filename), 'r', encoding='utf-8') as file:
values[filename] = file.read().strip()
except FileNotFoundError:
values[filename] = ''
# Get special values
values['DATE'] = datetime.now().strftime('%Y.%m.%d')
# Exclude containers
except_dirs = [] # relative path to dir from root
except_files = [] # relative path to file from root
except_names = [] # filename (with extension) anywhere
except_exts = [] # extension anywhere
# If 'exclude' is dictionary like object
try:
_empty = ()
# Excludes relative to root
for key, container in zip(('folders', 'files'),
(except_dirs, except_files)):
container.extend(os_path_join(infolder, p) for p in exclude.get(key, _empty))
# Excludes anywhere
for key, container in zip(('names', 'extensions'),
(except_names, except_exts)):
container.extend(exclude.get(key, _empty))
# If 'exclude' is an iterable object
except AttributeError:
except_names = exclude
# Include containers
permit_names = [] # filename (with extension) anywhere
permit_exts = [] # extension anywhere
# If 'include' is dictionary like object
try:
_empty = ()
# Includes anywhere
for key, container in zip(('names', 'extensions'),
(permit_names, permit_exts)):
container.extend(include.get(key, _empty))
# If 'include' is an iterable object
except AttributeError:
permit_names = include
# Walk through all files and folders in the passed folder
# FIXME: what if none of the files changed only INFO has been updated?
# Scan through all files and folders
with check_Checker(infolder, file='.clic_cache') as checker:
for root, dirs, filenames in os_walk(infolder):
# If skip this folder and all subfolders
if root in except_dirs:
dirs.clear()
continue
# Check all files in folder
for filename in filenames:
filepath = os_path_join(root, filename)[2:]
# If skip this exact file
if filepath in except_files:
continue
name, extension = os_path_splitext(filename)
# If file or extension is not banned and it is on the
# white-list and it changed since last time checked and
# this is not and overwrite-call
if (filename not in except_names and
extension not in except_exts and
(extension in permit_exts or
filename in permit_names) and
checker.ischanged(filepath) and
not overwrite):
values['SIZE'] = _size(os_path_getsize(filepath))
# FIXME: make it more generic than ./ -- what if ../../?
values['FILE'] = filepath[2:] if filepath.startswith('./') else filepath
values['FILE_NAME'] = file
values['FILE_BASE'] = name
if _comment(header.format(**values), filepath, pattern, align, width):
# Update checker after the file has been modified
checker.update()
# Report
print('CLIC: processed {!r}'.format(filepath))
def collect(infolder,
line = comment_LINE,
block = comment_BLOCK,
tags = WORDS,
marks = MARKS,
include=INCLUDE,
exclude=EXCLUDE,
overwrite=False):
# Process block comment marks
blocks_open, blocks_close = comment_block_comments(block)
# TODO: Make hidden files OS independent, probably using
# https://docs.python.org/3.4/library/tempfile.html ?
# FIXME: for some reason, if a comment-type ever existed in the TODO
# file, but after a while its posts are all gone, the keyword
# still remains there, according to the current TODO file,
# which still have the "QUESTIONS" keyword, and comment
# TODO: Add explicit-remove/browsing capabilities of the .*_cache files
# (for example: if git reverted changes --> remove hash from cache file)
# The best solution would be a complete CLI tool, to read and manage
# and use the cutils command line tools
# Compile regular expression patterns
pattern1 = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)),
blocks_open,
r'|'.join(map(comment_escape, tags)),
r'|'.join(map(comment_escape, marks)),
blocks_close),
flags=re_IGNORECASE | re_DOTALL | re_MULTILINE | re_VERBOSE)
pattern2 = re_compile(r'\n')
# Get previously generated collection of all posts
COLLECTED = os_path_join(infolder, '.ccom_todo')
try:
with open(COLLECTED, 'rb') as file:
collected = pickle_load(file)
except (FileNotFoundError, EOFError):
collected = table_Table(row=OrderedDict)
# Clear cache -- remove all non-existing files
for filepath in collected.rows():
if not os_path_isfile(filepath):
del collected[filepath]
# Exception containers
except_dirs = [] # relative path to dir from root
except_files = [] # relative path to file from root
except_names = [] # filename (with extension) anywhere
except_exts = [] # extension anywhere
# If 'exclude' is dictionary like object
try:
_empty = ()
# Exceptions relative to root
for key, container in zip(('folders', 'files'),
(except_dirs, except_files)):
container.extend(os_path_join(infolder, p) for p in exclude.get(key, _empty))
# Exceptions anywhere
for key, container in zip(('names', 'extensions'),
(except_names, except_exts)):
container.extend(exclude.get(key, _empty))
# If 'exclude' is an iterable object
except AttributeError:
except_names = exclude
# Include containers
permit_names = [] # filename (with extension) anywhere
permit_exts = [] # extension anywhere
# If 'include' is dictionary like object
try:
_empty = ()
# Includes anywhere
for key, container in zip(('names', 'extensions'),
(permit_names, permit_exts)):
container.extend(include.get(key, _empty))
# If 'include' is an iterable object
except AttributeError:
permit_names = include
# Scan through all files and folders
with check_Checker(infolder, file='.ccom_cache') as checker:
for root, dirs, filenames in os_walk(infolder):
# If skip this folder and all subfolders
if root in except_dirs:
dirs.clear()
continue
# Check all files in folder
for filename in filenames:
filepath = os_path_join(root, filename)[2:]
# If skip this exact file
if filepath in except_files:
continue
name, extension = os_path_splitext(filename)
# If file or extension is not banned and it is on the
# white-list and it changed since last time checked and
# this is not and overwrite-call
if (filename not in except_names and
extension not in except_exts and
(extension in permit_exts or filename in permit_names) and
checker.ischanged(filepath) and
not overwrite):
with open(filepath, encoding='utf-8') as file:
_search(collected, pattern1, pattern2,
file.read(), filepath, marks)
# Save collection of all posts
with open(COLLECTED, 'wb') as file:
pickle_dump(collected, file, pickle_HIGHEST_PROTOCOL)
# Open the todo file and write out the results
with open('TODO', 'w', encoding='utf-8') as todo:
# Make it compatible with cver.py
todo.write('## INFO ##\n'*2)
# Format TODO file as yaml
for key in itertools_chain(tags, marks.values()):
KEY = key.upper()
try:
types = collected[KEY].items()
len_pos = todo.tell()
# Offset for separator comment and
# leading and trailing new lines
todo.write(' '*82)
todo.write('{}:\n'.format(KEY))
index = 1
for filename, posts in types:
for i, (linenumber, content) in enumerate(posts, start=index):
todo.write(_ITEM.format(msg='\n'.join(content),
index=i,
short=_SHORT,
long=_SHORT*2,
sep='- '*38,
file=filename,
line=linenumber))
index = i + 1
todo.write('\n')
# Move back to tag separator comment
todo.seek(len_pos)
todo.write('\n#{:-^78}#\n'.format(
' {} POSTS IN {} FILES '.format(index - 1, len(types))))
# Move back to the end
todo.seek(0, 2)
except KeyError:
continue
print('CCOM: placed {!r}'.format(os_path_join(infolder, 'TODO')))
def document(infolder,
outfolder,
extension,
loader,
external_css=None,
generate_toc=None,
overwrite=False):
# Get previously generated TOC object
TOC = os_path_join(infolder, '.cdoc_toc')
try:
with open(TOC, 'rb') as file:
old_toc = pickle_load(file)
except (FileNotFoundError, EOFError):
old_toc = table_Dict2D(OrderedDict)
# Create new TOC object
new_toc = table_Dict2D(OrderedDict)
# TODO: do we really need a separate OrderedDict for pages ???
pages = OrderedDict()
anonym = iter_count()
# TODO: Create real dependency graphs
# Document object:
# parents = set() # other documents depending on this document
# children = set() # other documents this document depending on
#
# If document changed:
# set all parents of document => changed
#
# If any of its children changed:
# set all parents of child => changed
#
# -- The loop should check if a document's change flag has already
# been set. If not, hash file, and set flag, and notify all
# dependencies (parents)
# Load all pages
with check_Checker(infolder, file='.cdoc_cache',
lazy_update=True) as checker:
# Go through all files
for file in os_listdir(infolder):
# If file has the proper extension
if file.endswith(extension):
# Create full file path
filepath = os_path_join(infolder, file)
# If file has been changed since last check
if checker.ischanged(filepath) and not overwrite:
# Regenerate file
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
# If file hasn't been changed
else:
# If file has been cached before
try:
# Get previous infos
filename, depends = old_toc[filepath]
pagename = old_toc.otherkey(filepath)
pages[pagename] = None
# If any of the dependencies has changed
for dependency in depends:
if checker.ischanged(dependency) and not overwrite:
# Regenerate file
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
break
# If file is new and hasn't been cached before
except KeyError:
# Generate it for the first time
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
# Store new values
new_toc[pagename:filepath] = filename, depends
# If order changing, renaming, inserting, deleting, etc. happened
if set(old_toc) - set(new_toc):
for pagename, filepath in new_toc.keys():
if pages[pagename] is None:
_process(os_path_basename(filepath), filepath, pages, loader,
anonym)
# Write back TOC object
with open(TOC, 'wb') as file:
pickle_dump(new_toc, file, pickle_HIGHEST_PROTOCOL)
# Generate Table of Content?
if generate_toc is None:
generate_toc = len(new_toc) > 1
# Create documents
_build(pages, outfolder, generate_toc, new_toc, external_css)
def document(infolder, outfolder, extension, loader, external_css=None,
generate_toc=None, overwrite=False):
# Get previously generated TOC object
TOC = os_path_join(infolder, '.cdoc_toc')
try:
with open(TOC, 'rb') as file:
old_toc = pickle_load(file)
except (FileNotFoundError, EOFError):
old_toc = table_Dict2D(OrderedDict)
# Create new TOC object
new_toc = table_Dict2D(OrderedDict)
# TODO: do we really need a separate OrderedDict for pages ???
pages = OrderedDict()
anonym = iter_count()
# TODO: Create real dependency graphs
# Document object:
# parents = set() # other documents depending on this document
# children = set() # other documents this document depending on
#
# If document changed:
# set all parents of document => changed
#
# If any of its children changed:
# set all parents of child => changed
#
# -- The loop should check if a document's change flag has already
# been set. If not, hash file, and set flag, and notify all
# dependencies (parents)
# Load all pages
with check_Checker(infolder, file='.cdoc_cache', lazy_update=True) as checker:
# Go through all files
for file in os_listdir(infolder):
# If file has the proper extension
if file.endswith(extension):
# Create full file path
filepath = os_path_join(infolder, file)
# If file has been changed since last check
if checker.ischanged(filepath) and not overwrite:
# Regenerate file
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
# If file hasn't been changed
else:
# If file has been cached before
try:
# Get previous infos
filename, depends = old_toc[filepath]
pagename = old_toc.otherkey(filepath)
pages[pagename] = None
# If any of the dependencies has changed
for dependency in depends:
if checker.ischanged(dependency) and not overwrite:
# Regenerate file
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
break
# If file is new and hasn't been cached before
except KeyError:
# Generate it for the first time
filename, pagename, depends = \
_process(infolder, file, filepath, pages, loader, anonym)
# Store new values
new_toc[pagename:filepath] = filename, depends
# If order changing, renaming, inserting, deleting, etc. happened
if set(old_toc) - set(new_toc):
for pagename, filepath in new_toc.keys():
if pages[pagename] is None:
_process(os_path_basename(filepath), filepath, pages, loader, anonym)
# Write back TOC object
with open(TOC, 'wb') as file:
pickle_dump(new_toc, file, pickle_HIGHEST_PROTOCOL)
# Generate Table of Content?
if generate_toc is None:
generate_toc = len(new_toc) > 1
# Create documents
_build(pages, outfolder, generate_toc, new_toc, external_css)
def header(infolder,
line=comment_LINE,
block=comment_BLOCK,
include=INCLUDE,
exclude=EXCLUDE,
overwrite=False):
# Compile regular expression pattern to match in scanned files
pattern = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)),
*comment_block_comments(block)),
flags=re_DOTALL | re_VERBOSE | re_MULTILINE)
# Define default values
align = _FORMAT['CENTER']
width = 80
# Update values based on INFO file
values = {}
with open(os_path_join(infolder, 'INFO'), 'r', encoding='utf-8') as file:
header = file.read()
match = re_match(
r'\s*#\s*format\s+'
r'((?P<align>CENTER|LEFT|RIGHT)\s+)?'
r'(?P<width>\d+)?', header)
if match:
align, width = match.group('align', 'width')
align = _FORMAT.get(align, _FORMAT['CENTER'])
try:
width = int(width)
except TypeError:
pass
# Add leading and trailing empty line
header = '\n{}\n'.format(header[match.end():].strip())
# Get file contents of special files
for filename in _FILES:
try:
with open(os_path_join(infolder, filename), 'r',
encoding='utf-8') as file:
values[filename] = file.read().strip()
except FileNotFoundError:
values[filename] = ''
# Get special values
values['DATE'] = datetime.now().strftime('%Y.%m.%d')
# Exclude containers
except_dirs = [] # relative path to dir from root
except_files = [] # relative path to file from root
except_names = [] # filename (with extension) anywhere
except_exts = [] # extension anywhere
# If 'exclude' is dictionary like object
try:
_empty = ()
# Excludes relative to root
for key, container in zip(('folders', 'files'),
(except_dirs, except_files)):
container.extend(
os_path_join(infolder, p) for p in exclude.get(key, _empty))
# Excludes anywhere
for key, container in zip(('names', 'extensions'),
(except_names, except_exts)):
container.extend(exclude.get(key, _empty))
# If 'exclude' is an iterable object
except AttributeError:
except_names = exclude
# Include containers
permit_names = [] # filename (with extension) anywhere
permit_exts = [] # extension anywhere
# If 'include' is dictionary like object
try:
_empty = ()
# Includes anywhere
for key, container in zip(('names', 'extensions'),
(permit_names, permit_exts)):
container.extend(include.get(key, _empty))
# If 'include' is an iterable object
except AttributeError:
permit_names = include
# Walk through all files and folders in the passed folder
# FIXME: what if none of the files changed only INFO has been updated?
# Scan through all files and folders
with check_Checker(infolder, file='.clic_cache') as checker:
for root, dirs, filenames in os_walk(infolder):
# If skip this folder and all subfolders
if root in except_dirs:
dirs.clear()
continue
# Check all files in folder
for filename in filenames:
filepath = os_path_join(root, filename)[2:]
# If skip this exact file
if filepath in except_files:
continue
name, extension = os_path_splitext(filename)
# If file or extension is not banned and it is on the
# white-list and it changed since last time checked and
# this is not and overwrite-call
if (filename not in except_names
and extension not in except_exts and
(extension in permit_exts or filename in permit_names)
and checker.ischanged(filepath) and not overwrite):
values['SIZE'] = _size(os_path_getsize(filepath))
# FIXME: make it more generic than ./ -- what if ../../?
values['FILE'] = filepath[2:] if filepath.startswith(
'./') else filepath
values['FILE_NAME'] = file
values['FILE_BASE'] = name
if _comment(header.format(**values), filepath, pattern,
align, width):
# Update checker after the file has been modified
checker.update()
# Report
print('CLIC: processed {!r}'.format(filepath))