def __to_save_img_seq(self, video_capture: VideoCapture, video_name: str):
# Frame counter
i = 0
# To be 100% sure that we get the filename without the extension
video_name = os_path_splitext(video_name)[0]
image_sequence_dir = os_path_join(self.im_seqs_dir, video_name)
while video_capture.isOpened():
ret, frame = video_capture.read()
if ret:
os_makedirs(image_sequence_dir, exist_ok=True)
if video_name is None:
raise Exception(
'VideoHandlerError: if video_to_image_sequence receives a "save_sequences=True" then "video_name" must also receive a value. '
)
frame_name = '{0:06}'.format(i + 1) + '_' + video_name + '.jpg'
# Save image
cv2_imwrite(os_path_join(image_sequence_dir, frame_name),
frame)
# wait 1ms and make another check before 'breaking'
if cv2_waitKey(1) & 0xFF == ord('q'):
break
i += 1
else:
break
print('Total frames of sequence read:', i - 1)
def reconstruct_from_postings(output_path, stemming):
postings = glob(output_path + "\\{}\\*.pkl".format("WithStem" if stemming else "WithoutStem"), recursive=True)
reconstructed = set()
corpus_size = 0
total_length = 0
for posting in postings:
if "inverted_idx" not in posting:
splited_path = os_path_splitext(posting)
print(splited_path)
file = utils.load_obj(splited_path[0])
for doc_list in file.values():
for doc in doc_list:
doc_id = doc[0]
doc_length = doc[4]
if doc_id not in reconstructed:
reconstructed.add(doc_id)
total_length += doc_length
corpus_size += 1
return corpus_size, float(total_length) / corpus_size
def _process(infolder, file, filepath, pages, loader, counter):
with open(filepath, encoding='utf-8') as f:
depends = set()
pagedata = _import(loader(f.read()), infolder, loader, depends)
pagename = pagedata.get('PAGE', 'Document {}'.format(next(counter)))
filename = pagedata.get('NAME', os_path_splitext(file)[0]) + '.html'
pages[pagename] = pagedata
return filename, pagename, depends
def _process(infolder, file, filepath, pages, loader, counter):
with open(filepath, encoding='utf-8') as f:
depends = set()
pagedata = _import(loader(f.read()), infolder, loader, depends)
pagename = pagedata.get('PAGE', 'Document {}'.format(next(counter)))
filename = pagedata.get('NAME', os_path_splitext(file)[0]) + '.html'
pages[pagename] = pagedata
return filename, pagename, depends
def save_code(self, obj):
"""saves the code containing the passed in object"""
if obj is not None:
module_path, ext = os_path_splitext(getfile(obj))
code_file_path = module_path + '.py' # Should get the py file, not the pyc, if compiled.
code_file_copy_path = self.dir_path+self.divider+os_path_split(module_path)[1]+".pyb"
if not os_path_exists(code_file_copy_path):
copyfile(code_file_path, code_file_copy_path)
log_info("Saved code to: {0}".format(code_file_copy_path))
def save_code(self, obj):
"""saves the code containing the passed in object"""
if obj is not None:
module_path, ext = os_path_splitext(getfile(obj))
code_file_path = module_path + '.py' # Should get the py file, not the pyc, if compiled.
code_file_copy_path = self.dir_path + self.divider + os_path_split(
module_path)[1] + ".pyb"
if not os_path_exists(code_file_copy_path):
copyfile(code_file_path, code_file_copy_path)
log_info("Saved code to: {0}".format(code_file_copy_path))
def update_db(self):
DB_update_folder = DB_DIR+"/Updates";
if(not os_path_exists(DB_update_folder)): return;
# Get all files that match versioning
file_versions = [];
for file in os_listdir(DB_update_folder):
filepath = os_path_join(DB_update_folder, file);
# Try to get the file's name excluding extension (valid filename example: v0.0.0.sql)
version_string = Version.version_string(os_path_splitext(os_path_basename(filepath))[0]);
# Include only files with proper version names within update range
if(os_path_isfile(filepath) and version_string and self.version_is_usable(Version(version_string))):
file_versions.append({"path": filepath, "version": Version(version_string)});
file_versions.sort(key=lambda file_version : file_version["version"]);
for file in file_versions:
if(self.call_shell_command(["sudo", "mysql", "-u", "root", "<", file["path"]])):
raise Exception(f"Failed to update DB with file {file['path']}");
def header(infolder,
line=comment_LINE,
block=comment_BLOCK,
include=INCLUDE,
exclude=EXCLUDE,
overwrite=False):
# Compile regular expression pattern to match in scanned files
pattern = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)),
*comment_block_comments(block)),
flags=re_DOTALL | re_VERBOSE | re_MULTILINE)
# Define default values
align = _FORMAT['CENTER']
width = 80
# Update values based on INFO file
values = {}
with open(os_path_join(infolder, 'INFO'), 'r', encoding='utf-8') as file:
header = file.read()
match = re_match(r'\s*#\s*format\s+'
r'((?P<align>CENTER|LEFT|RIGHT)\s+)?'
r'(?P<width>\d+)?', header)
if match:
align, width = match.group('align', 'width')
align = _FORMAT.get(align, _FORMAT['CENTER'])
try:
width = int(width)
except TypeError:
pass
# Add leading and trailing empty line
header = '\n{}\n'.format(header[match.end():].strip())
# Get file contents of special files
for filename in _FILES:
try:
with open(os_path_join(infolder, filename), 'r', encoding='utf-8') as file:
values[filename] = file.read().strip()
except FileNotFoundError:
values[filename] = ''
# Get special values
values['DATE'] = datetime.now().strftime('%Y.%m.%d')
# Exclude containers
except_dirs = [] # relative path to dir from root
except_files = [] # relative path to file from root
except_names = [] # filename (with extension) anywhere
except_exts = [] # extension anywhere
# If 'exclude' is dictionary like object
try:
_empty = ()
# Excludes relative to root
for key, container in zip(('folders', 'files'),
(except_dirs, except_files)):
container.extend(os_path_join(infolder, p) for p in exclude.get(key, _empty))
# Excludes anywhere
for key, container in zip(('names', 'extensions'),
(except_names, except_exts)):
container.extend(exclude.get(key, _empty))
# If 'exclude' is an iterable object
except AttributeError:
except_names = exclude
# Include containers
permit_names = [] # filename (with extension) anywhere
permit_exts = [] # extension anywhere
# If 'include' is dictionary like object
try:
_empty = ()
# Includes anywhere
for key, container in zip(('names', 'extensions'),
(permit_names, permit_exts)):
container.extend(include.get(key, _empty))
# If 'include' is an iterable object
except AttributeError:
permit_names = include
# Walk through all files and folders in the passed folder
# FIXME: what if none of the files changed only INFO has been updated?
# Scan through all files and folders
with check_Checker(infolder, file='.clic_cache') as checker:
for root, dirs, filenames in os_walk(infolder):
# If skip this folder and all subfolders
if root in except_dirs:
dirs.clear()
continue
# Check all files in folder
for filename in filenames:
filepath = os_path_join(root, filename)[2:]
# If skip this exact file
if filepath in except_files:
continue
name, extension = os_path_splitext(filename)
# If file or extension is not banned and it is on the
# white-list and it changed since last time checked and
# this is not and overwrite-call
if (filename not in except_names and
extension not in except_exts and
(extension in permit_exts or
filename in permit_names) and
checker.ischanged(filepath) and
not overwrite):
values['SIZE'] = _size(os_path_getsize(filepath))
# FIXME: make it more generic than ./ -- what if ../../?
values['FILE'] = filepath[2:] if filepath.startswith('./') else filepath
values['FILE_NAME'] = file
values['FILE_BASE'] = name
if _comment(header.format(**values), filepath, pattern, align, width):
# Update checker after the file has been modified
checker.update()
# Report
print('CLIC: processed {!r}'.format(filepath))
def collect(infolder,
line = comment_LINE,
block = comment_BLOCK,
tags = WORDS,
marks = MARKS,
include=INCLUDE,
exclude=EXCLUDE,
overwrite=False):
# Process block comment marks
blocks_open, blocks_close = comment_block_comments(block)
# TODO: Make hidden files OS independent, probably using
# https://docs.python.org/3.4/library/tempfile.html ?
# FIXME: for some reason, if a comment-type ever existed in the TODO
# file, but after a while its posts are all gone, the keyword
# still remains there, according to the current TODO file,
# which still have the "QUESTIONS" keyword, and comment
# TODO: Add explicit-remove/browsing capabilities of the .*_cache files
# (for example: if git reverted changes --> remove hash from cache file)
# The best solution would be a complete CLI tool, to read and manage
# and use the cutils command line tools
# Compile regular expression patterns
pattern1 = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)),
blocks_open,
r'|'.join(map(comment_escape, tags)),
r'|'.join(map(comment_escape, marks)),
blocks_close),
flags=re_IGNORECASE | re_DOTALL | re_MULTILINE | re_VERBOSE)
pattern2 = re_compile(r'\n')
# Get previously generated collection of all posts
COLLECTED = os_path_join(infolder, '.ccom_todo')
try:
with open(COLLECTED, 'rb') as file:
collected = pickle_load(file)
except (FileNotFoundError, EOFError):
collected = table_Table(row=OrderedDict)
# Clear cache -- remove all non-existing files
for filepath in collected.rows():
if not os_path_isfile(filepath):
del collected[filepath]
# Exception containers
except_dirs = [] # relative path to dir from root
except_files = [] # relative path to file from root
except_names = [] # filename (with extension) anywhere
except_exts = [] # extension anywhere
# If 'exclude' is dictionary like object
try:
_empty = ()
# Exceptions relative to root
for key, container in zip(('folders', 'files'),
(except_dirs, except_files)):
container.extend(os_path_join(infolder, p) for p in exclude.get(key, _empty))
# Exceptions anywhere
for key, container in zip(('names', 'extensions'),
(except_names, except_exts)):
container.extend(exclude.get(key, _empty))
# If 'exclude' is an iterable object
except AttributeError:
except_names = exclude
# Include containers
permit_names = [] # filename (with extension) anywhere
permit_exts = [] # extension anywhere
# If 'include' is dictionary like object
try:
_empty = ()
# Includes anywhere
for key, container in zip(('names', 'extensions'),
(permit_names, permit_exts)):
container.extend(include.get(key, _empty))
# If 'include' is an iterable object
except AttributeError:
permit_names = include
# Scan through all files and folders
with check_Checker(infolder, file='.ccom_cache') as checker:
for root, dirs, filenames in os_walk(infolder):
# If skip this folder and all subfolders
if root in except_dirs:
dirs.clear()
continue
# Check all files in folder
for filename in filenames:
filepath = os_path_join(root, filename)[2:]
# If skip this exact file
if filepath in except_files:
continue
name, extension = os_path_splitext(filename)
# If file or extension is not banned and it is on the
# white-list and it changed since last time checked and
# this is not and overwrite-call
if (filename not in except_names and
extension not in except_exts and
(extension in permit_exts or filename in permit_names) and
checker.ischanged(filepath) and
not overwrite):
with open(filepath, encoding='utf-8') as file:
_search(collected, pattern1, pattern2,
file.read(), filepath, marks)
# Save collection of all posts
with open(COLLECTED, 'wb') as file:
pickle_dump(collected, file, pickle_HIGHEST_PROTOCOL)
# Open the todo file and write out the results
with open('TODO', 'w', encoding='utf-8') as todo:
# Make it compatible with cver.py
todo.write('## INFO ##\n'*2)
# Format TODO file as yaml
for key in itertools_chain(tags, marks.values()):
KEY = key.upper()
try:
types = collected[KEY].items()
len_pos = todo.tell()
# Offset for separator comment and
# leading and trailing new lines
todo.write(' '*82)
todo.write('{}:\n'.format(KEY))
index = 1
for filename, posts in types:
for i, (linenumber, content) in enumerate(posts, start=index):
todo.write(_ITEM.format(msg='\n'.join(content),
index=i,
short=_SHORT,
long=_SHORT*2,
sep='- '*38,
file=filename,
line=linenumber))
index = i + 1
todo.write('\n')
# Move back to tag separator comment
todo.seek(len_pos)
todo.write('\n#{:-^78}#\n'.format(
' {} POSTS IN {} FILES '.format(index - 1, len(types))))
# Move back to the end
todo.seek(0, 2)
except KeyError:
continue
print('CCOM: placed {!r}'.format(os_path_join(infolder, 'TODO')))
def header(infolder,
line=comment_LINE,
block=comment_BLOCK,
include=INCLUDE,
exclude=EXCLUDE,
overwrite=False):
# Compile regular expression pattern to match in scanned files
pattern = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)),
*comment_block_comments(block)),
flags=re_DOTALL | re_VERBOSE | re_MULTILINE)
# Define default values
align = _FORMAT['CENTER']
width = 80
# Update values based on INFO file
values = {}
with open(os_path_join(infolder, 'INFO'), 'r', encoding='utf-8') as file:
header = file.read()
match = re_match(
r'\s*#\s*format\s+'
r'((?P<align>CENTER|LEFT|RIGHT)\s+)?'
r'(?P<width>\d+)?', header)
if match:
align, width = match.group('align', 'width')
align = _FORMAT.get(align, _FORMAT['CENTER'])
try:
width = int(width)
except TypeError:
pass
# Add leading and trailing empty line
header = '\n{}\n'.format(header[match.end():].strip())
# Get file contents of special files
for filename in _FILES:
try:
with open(os_path_join(infolder, filename), 'r',
encoding='utf-8') as file:
values[filename] = file.read().strip()
except FileNotFoundError:
values[filename] = ''
# Get special values
values['DATE'] = datetime.now().strftime('%Y.%m.%d')
# Exclude containers
except_dirs = [] # relative path to dir from root
except_files = [] # relative path to file from root
except_names = [] # filename (with extension) anywhere
except_exts = [] # extension anywhere
# If 'exclude' is dictionary like object
try:
_empty = ()
# Excludes relative to root
for key, container in zip(('folders', 'files'),
(except_dirs, except_files)):
container.extend(
os_path_join(infolder, p) for p in exclude.get(key, _empty))
# Excludes anywhere
for key, container in zip(('names', 'extensions'),
(except_names, except_exts)):
container.extend(exclude.get(key, _empty))
# If 'exclude' is an iterable object
except AttributeError:
except_names = exclude
# Include containers
permit_names = [] # filename (with extension) anywhere
permit_exts = [] # extension anywhere
# If 'include' is dictionary like object
try:
_empty = ()
# Includes anywhere
for key, container in zip(('names', 'extensions'),
(permit_names, permit_exts)):
container.extend(include.get(key, _empty))
# If 'include' is an iterable object
except AttributeError:
permit_names = include
# Walk through all files and folders in the passed folder
# FIXME: what if none of the files changed only INFO has been updated?
# Scan through all files and folders
with check_Checker(infolder, file='.clic_cache') as checker:
for root, dirs, filenames in os_walk(infolder):
# If skip this folder and all subfolders
if root in except_dirs:
dirs.clear()
continue
# Check all files in folder
for filename in filenames:
filepath = os_path_join(root, filename)[2:]
# If skip this exact file
if filepath in except_files:
continue
name, extension = os_path_splitext(filename)
# If file or extension is not banned and it is on the
# white-list and it changed since last time checked and
# this is not and overwrite-call
if (filename not in except_names
and extension not in except_exts and
(extension in permit_exts or filename in permit_names)
and checker.ischanged(filepath) and not overwrite):
values['SIZE'] = _size(os_path_getsize(filepath))
# FIXME: make it more generic than ./ -- what if ../../?
values['FILE'] = filepath[2:] if filepath.startswith(
'./') else filepath
values['FILE_NAME'] = file
values['FILE_BASE'] = name
if _comment(header.format(**values), filepath, pattern,
align, width):
# Update checker after the file has been modified
checker.update()
# Report
print('CLIC: processed {!r}'.format(filepath))
