def is_cloned(
) -> bool: # pragma: no cover ## Only used by 1 thing for dev.
"""
Checks if the local version is a cloned (from git) one.
"""
file_helper = FileHelper()
directory_helper = DirectoryHelper()
if not directory_helper.set_path(".git").exists():
return False
list_of_files = [
".coveragerc",
".gitignore",
"CODE_OF_CONDUCT.rst",
"CONTRIBUTING.rst",
"MANIFEST.in",
"README.rst",
"requirements.txt",
"setup.py",
"version.yaml",
]
list_of_dirs = ["docs", "PyFunceble", "tests", ".github"]
if not all(file_helper.set_path(x).exists() for x in list_of_files):
return False
if not all(
directory_helper.set_path(x).exists() for x in list_of_dirs):
return False
return True
def setUp(self) -> None:
"""
Setups everything needed for the tests.
"""
self.temp_path = tempfile.TemporaryDirectory()
self.helper = DirectoryHelper(path=self.temp_path.name)
def get_backup_data(self) -> dict:
"""
Provides the data which acts as a backup.
"""
result = {}
base_dir = self.get_output_basedir()
file_helper = FileHelper()
hash_helper = HashHelper()
for file in DirectoryHelper(base_dir).list_all_files():
file_helper.set_path(file)
reduced_path = self.get_path_without_base_dir(file)
directory, filename = reduced_path.rsplit(os.sep, 1)
file_hash = hash_helper.hash_file(file_helper.path)
dataset = {
filename: {
"content": file_helper.read(),
"hash": file_hash
}
}
if directory not in result:
result[directory] = dataset
else:
result[directory].update(dataset)
PyFunceble.facility.Logger.debug("Dir Structure to backup:\n%r",
result)
return result
def authorized(self) -> bool:
return any(
FileHelper(os.path.join(self.info_manager.WORKSPACE_DIR,
x)).exists()
for x in self.FILES_TO_REMOVE) or any(
DirectoryHelper(
os.path.join(self.info_manager.WORKSPACE_DIR, x)).exists()
for x in self.DIRS_TO_REMOVE)
def get_output_directory() -> str: # pragma: no cover ## Not relevant
"""
Provides the location of the output directory.
"""
env_var_helper = EnvironmentVariableHelper()
directory_helper = DirectoryHelper()
if env_var_helper.set_name("PYFUNCEBLE_OUTPUT_LOCATION").exists():
output_directory = env_var_helper.get_value()
else:
output_directory = directory_helper.get_current(with_end_sep=True)
if not output_directory.endswith(os.sep):
output_directory += os.sep
return output_directory
def start(self) -> "OurInfrastructureUpdater":
file_helper = FileHelper()
dir_helper = DirectoryHelper()
for file in self.FILES_TO_REMOVE:
file_helper.set_path(
os.path.join(self.info_manager.WORKSPACE_DIR, file))
if file_helper.exists():
logging.info("Starting deletion of %r", file_helper.path)
file_helper.delete()
logging.info("Finished deletion of %r", file_helper.path)
for directory in self.DIRS_TO_REMOVE:
dir_helper.set_path(
os.path.join(self.info_manager.WORKSPACE_DIR, directory))
if dir_helper.exists():
logging.info("Starting deletion of %r", dir_helper.path)
dir_helper.delete()
logging.info("Finished deletion of %r", dir_helper.path)
return self
def pre(self) -> "PyFuncebleConfigLocationUpdater":
logging.info(
"Started maintenance of %r.",
self.info_manager.PYFUNCEBLE_CONFIG_DIR,
)
DirectoryHelper(self.info_manager.PYFUNCEBLE_CONFIG_DIR).create()
return self
def output_directory(self) -> Optional[str]:
"""
Provides the current state of the :code:`_output_directory` attribute.
"""
if self.authorized:
DirectoryHelper(self._output_directory).create()
return self._output_directory
def get_output_basedir(self) -> str:
"""
Provides the output base directory.
:param create_if_missing:
Authorizes the creation of the directory if it's missing.
"""
result = super().get_output_basedir()
if not DirectoryHelper(result).exists():
DirectoryStructureRestoration(self.parent_dirname).start()
return result
def test_set_path_through_init(self) -> None:
"""
Tests the overwritting of the path to work through the class
constructor.
"""
given = self.temp_path.name
expected = self.temp_path.name
helper = DirectoryHelper(given)
actual = helper.path
self.assertEqual(expected, actual)
def test_join_path(self) -> None:
"""
Tests the method which let us join paths.
"""
if PlatformUtility.is_windows():
given = "\\hello\\world"
expected = "\\hello\\world\\hello\\world"
else:
given = "/hello/world"
expected = "/hello/world/hello/world"
actual = DirectoryHelper(given).join_path("hello", "world")
self.assertEqual(expected, actual)
def start(self) -> "InfrastructureCleaner":
analytic_directory = DirectoryHelper(
os.path.join(outputs.OUTPUT_ROOT_DIRECTORY, "json"))
if analytic_directory.exists():
for element in os.listdir(outputs.OUTPUT_ROOT_DIRECTORY):
if any(x in element for x in self.STD_FILES_TO_IGNORE):
continue
dir_helper = DirectoryHelper(
os.path.join(outputs.OUTPUT_ROOT_DIRECTORY, element))
if dir_helper.exists():
dir_helper.delete()
logging.info("Deleted: %r", dir_helper.path)
del analytic_directory
for file in self.FILES_TO_REMOVE:
if not isinstance(file, list):
file_helper = FileHelper(
os.path.join(outputs.CURRENT_DIRECTORY, file))
else:
file_helper = FileHelper(
os.path.join(outputs.CURRENT_DIRECTORY, *file))
if file_helper.exists():
file_helper.delete()
logging.info("Deleted: %r", file_helper.path)
for file in self.FILES_TO_MOVE_TO_PYFUNCEBLE_CONFIG:
file_helper = FileHelper(
os.path.join(outputs.CURRENT_DIRECTORY, file))
if file_helper.exists():
file_helper.move(
os.path.join(outputs.PYFUNCEBLE_CONFIG_DIRECTORY, file))
logging.info("Moved: %r", file_helper.path)
return self
def authorized(self) -> bool:
if not DirectoryHelper(
self.info_manager.PYFUNCEBLE_CONFIG_DIR).exists():
return True
file_helper = FileHelper()
for file in self.FILES_TO_MOVE:
if file_helper.set_path(
os.path.join(self.info_manager.PYFUNCEBLE_CONFIG_DIR,
file)).exists():
return True
if all(
file_helper.set_path(
os.path.join(self.info_manager.PYFUNCEBLE_CONFIG_DIR,
x)).exists()
for x in self.INACTIVE_FILES_TO_DELETE):
return True
return False
def write(self,
data: Any,
*,
overwrite: bool = False,
encoding: str = "utf-8") -> "FileHelper":
"""
Write the given data into the given file path.
:param data: The data to write.
:param encoding: The encoding to use while opening the file.
"""
if overwrite or not self.exists():
DirectoryHelper(os.path.dirname(self.path)).create()
with self.open("w", encoding=encoding) as file_stream:
file_stream.write(data)
else:
with self.open("a", encoding=encoding) as file_stream:
file_stream.write(data)
return self
def update_documentation() -> "ProductionPrep":
"""
Updates the code documentation.
:raise RuntimeError:
When one of the wanted directory is not found.
"""
PyFunceble.facility.Logger.info(
"Started to update and generate the documentation.",
)
docs_dir_helper = DirectoryHelper("docs")
source_code_dir_helper = DirectoryHelper("PyFunceble")
if not docs_dir_helper.exists():
raise RuntimeError(f"{docs_dir_helper.realpath!r} not found.")
if not source_code_dir_helper.exists():
raise RuntimeError(f"{source_code_dir_helper.realpath!r} not found.")
header = "Code Documentation"
source_code_destination = os.path.join(docs_dir_helper.realpath, "code")
CommandHelper(
f"sphinx-apidoc -d 5 -f -H {header!r} -o "
f"{source_code_destination!r} {source_code_dir_helper.realpath}"
).execute(raise_on_error=True)
docs_destination = os.path.join(docs_dir_helper.realpath, "_build", "html")
CommandHelper(
f"sphinx-build -a -Q {docs_dir_helper.realpath!r} {docs_destination!r}"
).execute(raise_on_error=False)
PyFunceble.facility.Logger.info(
"Finished to update and generate the documentation.",
)
def generate_next_file(
directory_path: str,
filename: str,
format_to_apply: str,
input_files: List[str],
template: Optional[str] = None,
endline: Optional[str] = None,
) -> None:
"""
A general function which write into the next file.
:param directory_path:
The path of the directory to write into.
:param filename:
The path of the filename.
:param format_to_apply:
The format to apply to each line.
:param input_file:
The input file to read
:param template:
The template to write before starting to write each lines.
:param endline:
The last line to write.
"""
dir_helper = DirectoryHelper(directory_path)
if dir_helper.exists():
for root, _, files in os.walk(directory_path):
for file in files:
FileHelper(os.path.join(root, file)).delete()
else:
dir_helper.create()
i = 0
destination = None
template_written = False
for input_file in input_files:
with open(input_file, "r", encoding="utf-8") as file_stream:
for line in file_stream:
line = line.strip()
destination = os.path.join(directory_path, filename.format(i))
if not FileHelper(destination).exists():
logging.info("Started Generation of %r", destination)
with open(destination, "a+",
encoding="utf-8") as destination_file_stream:
if i == 0 and template and not template_written:
logging.debug("Writting template:\n%s", template)
destination_file_stream.write(template)
template_written = True
destination_file_stream.write(
f"{format_to_apply.format(line)}\n")
if destination_file_stream.tell(
) >= outputs.MAX_FILE_SIZE_IN_BYTES:
logging.info(
"Finished Generation of %r",
destination,
)
i += 1
continue
if destination and endline:
with open(destination, "a+",
encoding="utf-8") as destination_file_stream:
logging.debug("Writting last line:\n%r", endline)
destination_file_stream.write(endline + "\n")
def cleaner() -> None:
"""
Provides the CLI for the public file generation.
"""
PyFunceble.facility.ConfigLoader.start()
colorama.init(autoreset=True)
description = (
f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}PyFunceble Cleaner"
f"{colorama.Style.RESET_ALL} - "
"The cleaner of PyFunceble."
)
parser = argparse.ArgumentParser(
description=description,
epilog=PyFunceble.cli.storage.STD_EPILOG,
add_help=True,
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument(
"-f", "--file", type=str, help="Sets the file to cleanup the information for."
)
parser.add_argument(
"-a",
"--all",
action="store_true",
help="Authorizes the cleanup of everything that is not useful anymore.",
default=False,
)
args = parser.parse_args()
SystemIntegrator(args).start()
directories_to_ignore = ["__pyfunceble_origin__"]
utility = FilesystemCleanup()
print(PyFunceble.cli.utils.ascii_logo.get_home_representation())
to_cleanup = []
directory_helper = DirectoryHelper()
if args.file:
to_cleanup.append(get_destination_from_origin(args.file))
if (
args.all
and directory_helper.set_path(PyFunceble.cli.storage.OUTPUT_DIRECTORY).exists()
):
to_cleanup.extend(
[
x
for x in os.listdir(PyFunceble.cli.storage.OUTPUT_DIRECTORY)
if directory_helper.set_path(
os.path.join(PyFunceble.cli.storage.OUTPUT_DIRECTORY, x)
).exists()
]
)
for directory in to_cleanup:
if directory in directories_to_ignore:
continue
utility.set_parent_dirname(directory)
print(f"Started cleanup of {utility.get_output_basedir()}.", end=" ")
try:
utility.clean_output_files()
print(PyFunceble.cli.storage.DONE)
except: # pylint: disable=bare-except
print(PyFunceble.cli.storage.ERROR)
print(traceback.format_exc())
sys.exit(1)
if args.all:
utility.clean_database()
DEPLOYMENT_DIR = "to_deploy"
CLONE_DIR = "clones"
DEPLOY_FILES_TO_IGNORE = []
COMMIT_MESSAGE = "Sync actions."
REPOS_TO_IGNORE = [
".github",
"template",
"actions-sync",
"infrastructure-launcher",
"infrastructure-monitoring",
"dev-center",
]
if __name__ == "__main__":
dir_helper = DirectoryHelper()
file_helper = FileHelper()
gh = Github(USER_TOKEN)
if not dir_helper.set_path(CLONE_DIR).exists():
dir_helper.create()
dir_helper.set_path(DEPLOYMENT_DIR)
CommandHelper(f"git config --global user.email {USER_GIT_EMAIL!r}").execute()
CommandHelper(f"git config --global user.name {USER_GIT_NAME!r}").execute()
CommandHelper("git config --global push.default simple").execute()
CommandHelper("git config --global pull.rebase false").execute()
for repo in gh.get_organization(ORG_NAME).get_repos():
def restore_from_backup(self) -> "DirectoryStructureRestoration":
"""
Restores or reconstruct the output directory.
"""
# pylint: disable=too-many-locals
PyFunceble.facility.Logger.info(
"Started restoration of the directory structure")
backup = self.get_backup_data()
base_dir = self.get_output_basedir()
dir_helper = DirectoryHelper()
file_helper = FileHelper()
if dir_helper.set_path(base_dir).exists():
for root, _, files in os.walk(dir_helper.path):
reduced_path = self.get_path_without_base_dir(root)
if reduced_path not in backup and root != reduced_path:
dir_helper.set_path(root).delete()
PyFunceble.facility.Logger.debug(
"Added %r into the list of directories to delete. "
"Reason: not found in own dataset.",
root,
)
continue
for directory, files in backup.items():
dir_helper.set_path(os.path.join(base_dir, directory)).create()
for file, dataset in files.items():
file_full_path = os.path.join(dir_helper.path, file)
if (file == ".gitignore"
and PyFunceble.cli.storage.STD_PARENT_DIRNAME
not in file_full_path):
to_delete = file_full_path
file_helper.set_path(to_delete).delete()
PyFunceble.facility.Logger.debug(
"(If exists) Deleted: %r. Reason: We are going to "
"replace it with .gitkeep",
to_delete,
)
file_full_path = file_full_path.replace(
".gitignore", ".gitkeep")
file_helper.set_path(file_full_path)
if not file_helper.exists():
file_helper.write(dataset["content"], overwrite=True)
PyFunceble.facility.Logger.info(
"Finished restoration of the directory structure")
return self
class TestDirectoryHelper(unittest.TestCase):
"""
Tests of the directory helper.
"""
def setUp(self) -> None:
"""
Setups everything needed for the tests.
"""
self.temp_path = tempfile.TemporaryDirectory()
self.helper = DirectoryHelper(path=self.temp_path.name)
def tearDown(self) -> None:
"""
Destroys everything needed for the tests.
"""
del self.helper
self.temp_path.cleanup()
del self.temp_path
def test_set_path_return(self) -> None:
"""
Tests the response from the method which let us set the path to work
with.
"""
actual = self.helper.set_path(self.temp_path.name)
self.assertIsInstance(actual, DirectoryHelper)
def test_set_path_method(self) -> None:
"""
Tests the method which let us set the path to work with.
"""
given = self.temp_path.name
expected = self.temp_path.name
self.helper.set_path(given)
actual = self.helper.path
self.assertEqual(expected, actual)
def test_set_path_attribute(self) -> None:
"""
Tests overwritting of the :code:`path` attribute.
"""
given = self.temp_path.name
expected = self.temp_path.name
self.helper.path = given
actual = self.helper.path
self.assertEqual(expected, actual)
def test_set_path_through_init(self) -> None:
"""
Tests the overwritting of the path to work through the class
constructor.
"""
given = self.temp_path.name
expected = self.temp_path.name
helper = DirectoryHelper(given)
actual = helper.path
self.assertEqual(expected, actual)
def test_set_path_not_str(self) -> None:
"""
Tests the method which let us set the path to work with for the case
that it's not a string.
"""
given = ["Hello", "World"]
self.assertRaises(TypeError, lambda: self.helper.set_path(given))
def test_realpath(self) -> None:
"""
Tests the method which let us get the real path to the currently set
path.
"""
expected = os.path.realpath(self.temp_path.name)
actual = self.helper.realpath
self.assertEqual(expected, actual)
@unittest.mock.patch.object(os, "getcwd")
def test_get_current(self, getcwd_path: unittest.mock.MagicMock) -> None:
"""
Tests the method which let us get the current directory.
"""
getcwd_path.return_value = "/hello/world"
expected = "/hello/world"
actual = self.helper.get_current()
self.assertEqual(expected, actual)
@unittest.mock.patch.object(os, "getcwd")
def test_get_current_with_sep(
self, getcwd_path: unittest.mock.MagicMock) -> None:
"""
Tests the method which let us get the current directory for the case
that we want to have the directory separator at the end.
"""
getcwd_path.return_value = "/hello/world"
if PlatformUtility.is_windows():
expected = "/hello/world\\"
else:
expected = "/hello/world/"
actual = self.helper.get_current(with_end_sep=True)
self.assertEqual(expected, actual)
def test_join_path(self) -> None:
"""
Tests the method which let us join paths.
"""
if PlatformUtility.is_windows():
given = "\\hello\\world"
expected = "\\hello\\world\\hello\\world"
else:
given = "/hello/world"
expected = "/hello/world/hello/world"
actual = DirectoryHelper(given).join_path("hello", "world")
self.assertEqual(expected, actual)
def test_exists(self) -> None:
"""
Tests the method which let check if a directory exists.
"""
self.helper.set_path(self.helper.join_path(secrets.token_urlsafe(8)))
expected = False
actual = self.helper.exists()
self.assertEqual(expected, actual)
expected = True
os.makedirs(self.helper.path)
actual = self.helper.exists()
self.assertEqual(expected, actual)
def test_create_and_delete(self) -> None:
"""
Tests the methods which let us create and delete a directory.
"""
self.helper.set_path(self.helper.join_path(secrets.token_urlsafe(8)))
expected = False
actual = self.helper.exists()
self.assertEqual(expected, actual)
expected = True
actual = self.helper.create().exists()
self.assertEqual(expected, actual)
expected = False
actual = self.helper.delete().exists()
self.assertEqual(expected, actual)
def test_list_all_subdirectories(self) -> None:
"""
Tests the method which let us list all subdirectories.
"""
dirname = [secrets.token_hex(6) for _ in range(10)]
for directory in dirname:
self.helper.set_path(os.path.join(self.temp_path.name,
directory)).create()
self.helper.set_path(
os.path.join(self.temp_path.name, directory,
directory)).create()
self.helper.set_path(
os.path.join(self.temp_path.name, directory, directory,
directory)).create()
self.helper.set_path(self.temp_path.name)
expected = (ListHelper(
[os.path.join(self.temp_path.name, x) for x in dirname] +
[os.path.join(self.temp_path.name, x, x) for x in dirname] +
[os.path.join(self.temp_path.name, x, x, x)
for x in dirname]).remove_duplicates().sort().subject)
actual = self.helper.list_all_subdirectories()
self.assertEqual(expected, actual)
def test_list_all_files(self) -> None:
"""
Tests the method which let us list all subdirectories.
"""
dirname = [secrets.token_hex(6) for _ in range(10)]
filename = secrets.token_hex(6)
for directory in dirname:
self.helper.set_path(os.path.join(self.temp_path.name,
directory)).create()
with open(os.path.join(self.helper.path, filename),
"w",
encoding="utf-8") as file_stream:
file_stream.write("Hello")
self.helper.set_path(
os.path.join(self.temp_path.name, directory,
directory)).create()
with open(os.path.join(self.helper.path, filename),
"w",
encoding="utf-8") as file_stream:
file_stream.write("Hello")
self.helper.set_path(
os.path.join(self.temp_path.name, directory, directory,
directory)).create()
with open(os.path.join(self.helper.path, filename),
"w",
encoding="utf-8") as file_stream:
file_stream.write("Hello")
self.helper.set_path(self.temp_path.name)
expected = (ListHelper(
[os.path.join(self.temp_path.name, x, filename)
for x in dirname] + [
os.path.join(self.temp_path.name, x, x, filename)
for x in dirname
] + [
os.path.join(self.temp_path.name, x, x, x, filename)
for x in dirname
]).sort().subject)
actual = self.helper.list_all_files()
self.assertEqual(expected, actual)
def get_config_directory(
*, project_name: str,
project_version: str) -> str: # pragma: no cover ## Not relevant
"""
Provides the location of the configuration directory.
"""
# pylint: disable=too-many-branches
env_var_helper = EnvironmentVariableHelper()
directory_helper = DirectoryHelper()
if env_var_helper.set_name("PYFUNCEBLE_CONFIG_DIR").exists():
config_directory = env_var_helper.get_value()
elif env_var_helper.set_name("PYFUNCEBLE_OUTPUT_DIR").exists():
config_directory = env_var_helper.get_value()
elif (VersionUtility(project_version).is_cloned()
or env_var_helper.set_name("TRAVIS_BUILD_DIR").exists()
or env_var_helper.set_name("CI_PROJECT_DIR").exists()
and env_var_helper.set_name("GITLAB_CI").exists()):
config_directory = directory_helper.get_current(with_end_sep=True)
else:
if PlatformUtility.is_unix():
config_dir_path = os.path.expanduser(os.path.join("~", ".config"))
if directory_helper.set_path(config_dir_path).exists():
config_directory = config_dir_path
elif directory_helper.set_path(os.path.expanduser("~")).exists():
config_directory = directory_helper.join_path(".")
else:
config_directory = directory_helper.get_current(
with_end_sep=True)
elif PlatformUtility.is_windows():
if env_var_helper.set_name("APPDATA").exists():
config_directory = env_var_helper.get_value()
else:
config_directory = directory_helper.get_current(
with_end_sep=True)
else:
config_directory = directory_helper.get_current(with_end_sep=True)
if not config_directory.endswith(os.sep):
config_directory += os.sep
config_directory += project_name + os.sep
if not directory_helper.set_path(config_directory).exists():
directory_helper.create()
if not config_directory.endswith(os.sep):
config_directory += os.sep
return config_directory
def cleanup_data_dir() -> None:
"""
Cleanup our data directory on shutdown.
"""
DirectoryHelper(PyFunceble.storage.CONFIG_DIRECTORY).delete()
from pyfunceble_webworker.routes.v1.api import api_router as v1_api_router
env_var_helper = EnvironmentVariableHelper()
if not env_var_helper.set_name("PYFUNCEBLE_WORKERS_DATA_DIR").exists():
raise RuntimeError(
"Could not find PYFUNCEBLE_WORKERS_DATA_DIR environment variable.")
pyfunceble_webworker.storage.CONFIG_DIRECTORY = env_var_helper.get_value()
PyFunceble.storage.CONFIG_DIRECTORY = os.path.join(
pyfunceble_webworker.storage.CONFIG_DIRECTORY,
secrets.token_hex(8),
)
DirectoryHelper(PyFunceble.storage.CONFIG_DIRECTORY).create()
DirectoryHelper(pyfunceble_webworker.storage.CONFIG_DIRECTORY).create()
file_helper = FileHelper()
pyfunceble_config_loader = ConfigLoader()
if file_helper.set_path(
os.path.join(
pyfunceble_webworker.storage.CONFIG_DIRECTORY,
assets_defaults.OVERWRITE_CONFIG_FILE,
)).exists():
local = DictHelper().from_yaml_file(file_helper.path)
if local:
pyfunceble_config_loader.custom_config = local
else: