def remove_duplicates(cl: ChangeLog, directory: str, dry_run: bool,
recursive: bool):
"""
Find and remove file duplicates
:param cl: ChangeLog instance
:param directory: directory to search
:param dry_run: True will not apply changes, only log them
:param recursive: True to recursively consider sub-directories
"""
bytes_freed = 0
# Generate a dictionary of duplicate files
file_map = find_duplicates(directory, recursive)
# Remove all files but the one with the shortest file name
for path, duplicates in file_map.items():
# Determine the shortest file name
shortest, others = shortest_filenames((duplicates + [path]))
chosen_name = shortest[0] if len(shortest) == 1 else \
get_most_recent_file(shortest)
LOG.info('"{}": {} duplicates found'.format(chosen_name, len(others)))
# Remove duplicate files
for duplicate in others:
LOG.info('"{}": remove duplicate "{}"'.format(
chosen_name, duplicate))
if not dry_run:
try:
bytes_freed += os.stat(duplicate).st_size
if os.path.exists(duplicate):
os.remove(duplicate)
cl.addChange(__name__,
True,
path=duplicate,
original=chosen_name)
else:
LOG.error(f'"{chosen_name}": duplicate does not exist')
cl.addChange(__name__,
False,
path=duplicate,
original=chosen_name,
message='duplicate does not exist')
except OSError as e:
LOG.error(f'"{chosen_name}": failed to remove '
f'"{duplicate}": {str(e)}')
cl.addChange(__name__,
False,
path=duplicate,
original=chosen_name,
message=str(e),
errno=e.errno)
else:
cl.addChange(__name__,
False,
path=duplicate,
original=chosen_name)
return bytes_freed
def write_change_log(readme: Path, change_log: ChangeLog):
content = open(readme, "r").read()
with open(readme, "w") as f:
if change_log.changes_available:
compiled_change_log = change_log.compile()
f.write(compiled_change_log)
f.write(content)
def update_or_create_item(source_dir: Path, target: Path, temp_docs: Path,
change_log: ChangeLog):
# Copy source directories to target directories, if target already has the directory, archive previous version
source_yaml = yaml.load(open(source_dir / "item.yaml", "r"),
Loader=yaml.FullLoader)
source_version = source_yaml["version"]
target_dir = target / source_dir.stem
target_latest = target_dir / "latest"
target_version = target_dir / source_version
html_dir = temp_docs / "_build"
html_file_name = f"{source_dir.stem}.html"
html_path = html_dir / html_file_name
update_html_resource_paths(html_path, relative_path="../../_static/")
# If its the first source is encountered, copy source to target
if not target_dir.exists():
copy_file(source_dir, target_latest)
copy_file(source_dir, target_version)
copy_file(html_path, target_latest / html_file_name)
copy_file(html_path, target_version / html_file_name)
change_log.new_item(source_dir.stem, source_version)
if target_version.exists() or source_version == target_version:
return
rmtree(target_latest)
copy_file(source_dir, target_latest)
copy_file(source_dir, target_version)
copy_file(html_path, target_latest / html_file_name)
copy_file(html_path, target_version / html_file_name)
change_log.update_item(source_dir.stem, source_version,
target_version.name)
def build(self):
# Document modules from source dir into self.temp_docs, remove temp conf.py, replace with template
apidoc_command = f"-F -o {self.temp_docs} {self.source}".split(" ")
print(f"Running sphinx api doc... [{apidoc_command}]")
sphinx_apidoc_cmd(apidoc_command)
conf_py_target = str(self.temp_docs / "conf.py")
unlink(conf_py_target)
data = asdict(
SphinxKWArgs(
sphinx_docs_target=str(self.source),
project_name="",
copyright="",
author="",
))
print("Rendering conf.py...")
render_jinja_file(str(TEMPLATES_DIR / "conf.template"), conf_py_target,
data)
print("Replacing .rst files with available notebooks...")
for notebook in self.notebook_path_iterator:
unlink(self.temp_docs / f"{notebook.stem}.rst", ignore_errors=True)
copy_file(source=notebook, target=(self.temp_docs / notebook.name))
build_command = f"-b html {self.temp_docs} {self.temp_docs / '_build'}"
print(f"Running sphinx build... [{build_command}]")
sphinx_build_cmd(build_command.split(" "))
change_log = ChangeLog()
for source_dir in self.source_item_iterator:
update_or_create_item(source_dir, self.target, self.temp_docs,
change_log)
target_static = self.target / "_static/_static"
if not target_static.exists():
copy_file(self.temp_docs / "_build/_static", target_static)
build_catalog_json(self.target_item_iterator, self.target)
render_index(self.target_item_iterator, self.target)
render_pages(self.target_item_iterator, self.target / "_static")
copy_file(STATIC_DIR / "styles.css",
self.target / "_static" / "styles.css")
write_change_log(self.target / "README.md", change_log)
rmtree(str(self.temp_docs))
def rename_files(cl: ChangeLog, directory: str, files: list, dry_run: bool,
style=None, space_char=None):
"""
Check file names for inconsistency and rename
:param cl: ChangeLog instance
:param directory: directory the files are contained in
:param files: a list of file names
:param dry_run: True will not apply changes, only log them
:param style: enforce a particular naming style
(CAPITALIZED, TITLECASE, LOWERCASE, UPPERCASE)
:param space_char: replace spaces in a file name with this character
"""
changes = check_files(files, style=style, space_char=space_char)
# iterate and apply changes
for of, nf in changes.items():
path = os.path.join(directory, of)
dest = os.path.join(directory, nf)
LOG.info('"{}": rename "{}"'.format(path, nf))
if not dry_run:
try:
if not os.path.exists(dest):
os.rename(path, dest)
cl.addChange(__name__, True, src=path, dest=dest)
else:
LOG.warning('"{}": destination already exists'.format(path))
cl.addChange(__name__,
False,
src=path,
dest=dest,
message='destination already exists')
except OSError as e:
LOG.error('failed to rename "{}": {}'.format(path, str(e)))
cl.addChange(__name__,
False,
src=path,
dest=dest,
message=str(e),
errno=e.errno)
else:
cl.addChange(__name__, False, src=path, dest=dest)
recursive = apr.recursive
style_text = apr.style[0] if apr.style is not None else None
space_char = apr.space_char[0] if apr.space_char is not None else None
if space_char is not None and len(space_char) > 1:
LOG.error('space character argument cannot be more than one character')
exit(10)
if dry_run:
LOG.info('dry run enabled')
if recursive:
LOG.info('recursive search enabled')
# prepare a new instance of a ChangeLog to record changes to files
cl = ChangeLog()
# record time before starting the operations
start = datetime.datetime.now()
stopwatch = timing_counter()
operations = operations_text.split(',')
bytes_freed = 0
valid_targets = []
for target in targets:
if not os.path.isdir(target):
LOG.error(f'invalid target "{target}": not a directory')
else:
valid_targets.append(target)
def remove_empty(cl: ChangeLog,
directory,
dry_run,
recursive):
for cd, dirs, files in os.walk(directory, followlinks=False):
files = [os.path.join(cd, f) for f in files]
for file in files:
size = os.stat(file).st_size
if size == 0:
LOG.info(f'remove empty file "{file}"')
if not dry_run:
try:
os.remove(file)
cl.addChange(__name__,
True,
path=file)
except OSError as e:
LOG.error(f'failed to remove "{file}": {str(e)}')
cl.addChange(__name__,
False,
path=file,
message=str(e),
errno=e.errno)
else:
cl.addChange(__name__,
False,
path=file)
for sd in dirs:
sd_path = os.path.join(cd, sd)
if os.path.exists(sd_path):
content = []
try:
content = os.listdir(sd_path)
except OSError as e:
LOG.error(f'failed to list directory "{sd_path}": {str(e)}')
cl.addChange(__name__,
False,
path=sd_path,
message=str(e),
errno=e.errno)
if len(content) == 0:
LOG.info(f'remove empty directory "{sd_path}"')
if not dry_run:
try:
os.rmdir(sd_path)
cl.addChange(__name__,
True,
path=sd_path)
except OSError as e:
LOG.error(f'failed to remove "{sd_path}": {str(e)}')
cl.addChange(__name__,
False,
path=sd_path,
message=str(e),
errno=e.errno)
else:
cl.addChange(__name__,
False,
path=sd_path)
if not recursive:
break