def __to_save_img_seq(self, video_capture: VideoCapture, video_name: str): # Frame counter i = 0 # To be 100% sure that we get the filename without the extension video_name = os_path_splitext(video_name)[0] image_sequence_dir = os_path_join(self.im_seqs_dir, video_name) while video_capture.isOpened(): ret, frame = video_capture.read() if ret: os_makedirs(image_sequence_dir, exist_ok=True) if video_name is None: raise Exception( 'VideoHandlerError: if video_to_image_sequence receives a "save_sequences=True" then "video_name" must also receive a value. ' ) frame_name = '{0:06}'.format(i + 1) + '_' + video_name + '.jpg' # Save image cv2_imwrite(os_path_join(image_sequence_dir, frame_name), frame) # wait 1ms and make another check before 'breaking' if cv2_waitKey(1) & 0xFF == ord('q'): break i += 1 else: break print('Total frames of sequence read:', i - 1)
def reconstruct_from_postings(output_path, stemming): postings = glob(output_path + "\\{}\\*.pkl".format("WithStem" if stemming else "WithoutStem"), recursive=True) reconstructed = set() corpus_size = 0 total_length = 0 for posting in postings: if "inverted_idx" not in posting: splited_path = os_path_splitext(posting) print(splited_path) file = utils.load_obj(splited_path[0]) for doc_list in file.values(): for doc in doc_list: doc_id = doc[0] doc_length = doc[4] if doc_id not in reconstructed: reconstructed.add(doc_id) total_length += doc_length corpus_size += 1 return corpus_size, float(total_length) / corpus_size
def _process(infolder, file, filepath, pages, loader, counter): with open(filepath, encoding='utf-8') as f: depends = set() pagedata = _import(loader(f.read()), infolder, loader, depends) pagename = pagedata.get('PAGE', 'Document {}'.format(next(counter))) filename = pagedata.get('NAME', os_path_splitext(file)[0]) + '.html' pages[pagename] = pagedata return filename, pagename, depends
def save_code(self, obj): """saves the code containing the passed in object""" if obj is not None: module_path, ext = os_path_splitext(getfile(obj)) code_file_path = module_path + '.py' # Should get the py file, not the pyc, if compiled. code_file_copy_path = self.dir_path+self.divider+os_path_split(module_path)[1]+".pyb" if not os_path_exists(code_file_copy_path): copyfile(code_file_path, code_file_copy_path) log_info("Saved code to: {0}".format(code_file_copy_path))
def save_code(self, obj): """saves the code containing the passed in object""" if obj is not None: module_path, ext = os_path_splitext(getfile(obj)) code_file_path = module_path + '.py' # Should get the py file, not the pyc, if compiled. code_file_copy_path = self.dir_path + self.divider + os_path_split( module_path)[1] + ".pyb" if not os_path_exists(code_file_copy_path): copyfile(code_file_path, code_file_copy_path) log_info("Saved code to: {0}".format(code_file_copy_path))
def update_db(self): DB_update_folder = DB_DIR+"/Updates"; if(not os_path_exists(DB_update_folder)): return; # Get all files that match versioning file_versions = []; for file in os_listdir(DB_update_folder): filepath = os_path_join(DB_update_folder, file); # Try to get the file's name excluding extension (valid filename example: v0.0.0.sql) version_string = Version.version_string(os_path_splitext(os_path_basename(filepath))[0]); # Include only files with proper version names within update range if(os_path_isfile(filepath) and version_string and self.version_is_usable(Version(version_string))): file_versions.append({"path": filepath, "version": Version(version_string)}); file_versions.sort(key=lambda file_version : file_version["version"]); for file in file_versions: if(self.call_shell_command(["sudo", "mysql", "-u", "root", "<", file["path"]])): raise Exception(f"Failed to update DB with file {file['path']}");
def header(infolder, line=comment_LINE, block=comment_BLOCK, include=INCLUDE, exclude=EXCLUDE, overwrite=False): # Compile regular expression pattern to match in scanned files pattern = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)), *comment_block_comments(block)), flags=re_DOTALL | re_VERBOSE | re_MULTILINE) # Define default values align = _FORMAT['CENTER'] width = 80 # Update values based on INFO file values = {} with open(os_path_join(infolder, 'INFO'), 'r', encoding='utf-8') as file: header = file.read() match = re_match(r'\s*#\s*format\s+' r'((?P<align>CENTER|LEFT|RIGHT)\s+)?' r'(?P<width>\d+)?', header) if match: align, width = match.group('align', 'width') align = _FORMAT.get(align, _FORMAT['CENTER']) try: width = int(width) except TypeError: pass # Add leading and trailing empty line header = '\n{}\n'.format(header[match.end():].strip()) # Get file contents of special files for filename in _FILES: try: with open(os_path_join(infolder, filename), 'r', encoding='utf-8') as file: values[filename] = file.read().strip() except FileNotFoundError: values[filename] = '' # Get special values values['DATE'] = datetime.now().strftime('%Y.%m.%d') # Exclude containers except_dirs = [] # relative path to dir from root except_files = [] # relative path to file from root except_names = [] # filename (with extension) anywhere except_exts = [] # extension anywhere # If 'exclude' is dictionary like object try: _empty = () # Excludes relative to root for key, container in zip(('folders', 'files'), (except_dirs, except_files)): container.extend(os_path_join(infolder, p) for p in exclude.get(key, _empty)) # Excludes anywhere for key, container in zip(('names', 'extensions'), (except_names, except_exts)): container.extend(exclude.get(key, _empty)) # If 'exclude' is an iterable object except AttributeError: except_names = exclude # Include containers permit_names = [] # filename (with extension) anywhere permit_exts = [] # extension anywhere # If 'include' is dictionary like object try: _empty = () # Includes anywhere for key, container in zip(('names', 'extensions'), (permit_names, permit_exts)): container.extend(include.get(key, _empty)) # If 'include' is an iterable object except AttributeError: permit_names = include # Walk through all files and folders in the passed folder # FIXME: what if none of the files changed only INFO has been updated? # Scan through all files and folders with check_Checker(infolder, file='.clic_cache') as checker: for root, dirs, filenames in os_walk(infolder): # If skip this folder and all subfolders if root in except_dirs: dirs.clear() continue # Check all files in folder for filename in filenames: filepath = os_path_join(root, filename)[2:] # If skip this exact file if filepath in except_files: continue name, extension = os_path_splitext(filename) # If file or extension is not banned and it is on the # white-list and it changed since last time checked and # this is not and overwrite-call if (filename not in except_names and extension not in except_exts and (extension in permit_exts or filename in permit_names) and checker.ischanged(filepath) and not overwrite): values['SIZE'] = _size(os_path_getsize(filepath)) # FIXME: make it more generic than ./ -- what if ../../? values['FILE'] = filepath[2:] if filepath.startswith('./') else filepath values['FILE_NAME'] = file values['FILE_BASE'] = name if _comment(header.format(**values), filepath, pattern, align, width): # Update checker after the file has been modified checker.update() # Report print('CLIC: processed {!r}'.format(filepath))
def collect(infolder, line = comment_LINE, block = comment_BLOCK, tags = WORDS, marks = MARKS, include=INCLUDE, exclude=EXCLUDE, overwrite=False): # Process block comment marks blocks_open, blocks_close = comment_block_comments(block) # TODO: Make hidden files OS independent, probably using # https://docs.python.org/3.4/library/tempfile.html ? # FIXME: for some reason, if a comment-type ever existed in the TODO # file, but after a while its posts are all gone, the keyword # still remains there, according to the current TODO file, # which still have the "QUESTIONS" keyword, and comment # TODO: Add explicit-remove/browsing capabilities of the .*_cache files # (for example: if git reverted changes --> remove hash from cache file) # The best solution would be a complete CLI tool, to read and manage # and use the cutils command line tools # Compile regular expression patterns pattern1 = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)), blocks_open, r'|'.join(map(comment_escape, tags)), r'|'.join(map(comment_escape, marks)), blocks_close), flags=re_IGNORECASE | re_DOTALL | re_MULTILINE | re_VERBOSE) pattern2 = re_compile(r'\n') # Get previously generated collection of all posts COLLECTED = os_path_join(infolder, '.ccom_todo') try: with open(COLLECTED, 'rb') as file: collected = pickle_load(file) except (FileNotFoundError, EOFError): collected = table_Table(row=OrderedDict) # Clear cache -- remove all non-existing files for filepath in collected.rows(): if not os_path_isfile(filepath): del collected[filepath] # Exception containers except_dirs = [] # relative path to dir from root except_files = [] # relative path to file from root except_names = [] # filename (with extension) anywhere except_exts = [] # extension anywhere # If 'exclude' is dictionary like object try: _empty = () # Exceptions relative to root for key, container in zip(('folders', 'files'), (except_dirs, except_files)): container.extend(os_path_join(infolder, p) for p in exclude.get(key, _empty)) # Exceptions anywhere for key, container in zip(('names', 'extensions'), (except_names, except_exts)): container.extend(exclude.get(key, _empty)) # If 'exclude' is an iterable object except AttributeError: except_names = exclude # Include containers permit_names = [] # filename (with extension) anywhere permit_exts = [] # extension anywhere # If 'include' is dictionary like object try: _empty = () # Includes anywhere for key, container in zip(('names', 'extensions'), (permit_names, permit_exts)): container.extend(include.get(key, _empty)) # If 'include' is an iterable object except AttributeError: permit_names = include # Scan through all files and folders with check_Checker(infolder, file='.ccom_cache') as checker: for root, dirs, filenames in os_walk(infolder): # If skip this folder and all subfolders if root in except_dirs: dirs.clear() continue # Check all files in folder for filename in filenames: filepath = os_path_join(root, filename)[2:] # If skip this exact file if filepath in except_files: continue name, extension = os_path_splitext(filename) # If file or extension is not banned and it is on the # white-list and it changed since last time checked and # this is not and overwrite-call if (filename not in except_names and extension not in except_exts and (extension in permit_exts or filename in permit_names) and checker.ischanged(filepath) and not overwrite): with open(filepath, encoding='utf-8') as file: _search(collected, pattern1, pattern2, file.read(), filepath, marks) # Save collection of all posts with open(COLLECTED, 'wb') as file: pickle_dump(collected, file, pickle_HIGHEST_PROTOCOL) # Open the todo file and write out the results with open('TODO', 'w', encoding='utf-8') as todo: # Make it compatible with cver.py todo.write('## INFO ##\n'*2) # Format TODO file as yaml for key in itertools_chain(tags, marks.values()): KEY = key.upper() try: types = collected[KEY].items() len_pos = todo.tell() # Offset for separator comment and # leading and trailing new lines todo.write(' '*82) todo.write('{}:\n'.format(KEY)) index = 1 for filename, posts in types: for i, (linenumber, content) in enumerate(posts, start=index): todo.write(_ITEM.format(msg='\n'.join(content), index=i, short=_SHORT, long=_SHORT*2, sep='- '*38, file=filename, line=linenumber)) index = i + 1 todo.write('\n') # Move back to tag separator comment todo.seek(len_pos) todo.write('\n#{:-^78}#\n'.format( ' {} POSTS IN {} FILES '.format(index - 1, len(types)))) # Move back to the end todo.seek(0, 2) except KeyError: continue print('CCOM: placed {!r}'.format(os_path_join(infolder, 'TODO')))
def header(infolder, line=comment_LINE, block=comment_BLOCK, include=INCLUDE, exclude=EXCLUDE, overwrite=False): # Compile regular expression pattern to match in scanned files pattern = re_compile(_COMMENT.format(r'|'.join(map(comment_escape, line)), *comment_block_comments(block)), flags=re_DOTALL | re_VERBOSE | re_MULTILINE) # Define default values align = _FORMAT['CENTER'] width = 80 # Update values based on INFO file values = {} with open(os_path_join(infolder, 'INFO'), 'r', encoding='utf-8') as file: header = file.read() match = re_match( r'\s*#\s*format\s+' r'((?P<align>CENTER|LEFT|RIGHT)\s+)?' r'(?P<width>\d+)?', header) if match: align, width = match.group('align', 'width') align = _FORMAT.get(align, _FORMAT['CENTER']) try: width = int(width) except TypeError: pass # Add leading and trailing empty line header = '\n{}\n'.format(header[match.end():].strip()) # Get file contents of special files for filename in _FILES: try: with open(os_path_join(infolder, filename), 'r', encoding='utf-8') as file: values[filename] = file.read().strip() except FileNotFoundError: values[filename] = '' # Get special values values['DATE'] = datetime.now().strftime('%Y.%m.%d') # Exclude containers except_dirs = [] # relative path to dir from root except_files = [] # relative path to file from root except_names = [] # filename (with extension) anywhere except_exts = [] # extension anywhere # If 'exclude' is dictionary like object try: _empty = () # Excludes relative to root for key, container in zip(('folders', 'files'), (except_dirs, except_files)): container.extend( os_path_join(infolder, p) for p in exclude.get(key, _empty)) # Excludes anywhere for key, container in zip(('names', 'extensions'), (except_names, except_exts)): container.extend(exclude.get(key, _empty)) # If 'exclude' is an iterable object except AttributeError: except_names = exclude # Include containers permit_names = [] # filename (with extension) anywhere permit_exts = [] # extension anywhere # If 'include' is dictionary like object try: _empty = () # Includes anywhere for key, container in zip(('names', 'extensions'), (permit_names, permit_exts)): container.extend(include.get(key, _empty)) # If 'include' is an iterable object except AttributeError: permit_names = include # Walk through all files and folders in the passed folder # FIXME: what if none of the files changed only INFO has been updated? # Scan through all files and folders with check_Checker(infolder, file='.clic_cache') as checker: for root, dirs, filenames in os_walk(infolder): # If skip this folder and all subfolders if root in except_dirs: dirs.clear() continue # Check all files in folder for filename in filenames: filepath = os_path_join(root, filename)[2:] # If skip this exact file if filepath in except_files: continue name, extension = os_path_splitext(filename) # If file or extension is not banned and it is on the # white-list and it changed since last time checked and # this is not and overwrite-call if (filename not in except_names and extension not in except_exts and (extension in permit_exts or filename in permit_names) and checker.ischanged(filepath) and not overwrite): values['SIZE'] = _size(os_path_getsize(filepath)) # FIXME: make it more generic than ./ -- what if ../../? values['FILE'] = filepath[2:] if filepath.startswith( './') else filepath values['FILE_NAME'] = file values['FILE_BASE'] = name if _comment(header.format(**values), filepath, pattern, align, width): # Update checker after the file has been modified checker.update() # Report print('CLIC: processed {!r}'.format(filepath))