def pyfs_scratch():
# home_fs = OSFS("~/")
home_fs = OSFS("/")
# home_fs = OSFS("C:/")
print(home_fs.listdir("../"))
# print(home_fs.tree(max_levels = 0, dirs_first = True))
dir_obj_infos = home_fs.scandir("../")
for info in dir_obj_infos:
if info.is_dir:
print("[dir] {}".format(info.name))
else:
print("[file] {} {} MB".format(info.name,
info.size / float(1024 * 1024)))
#!/usr/bin/env python3
import pendulum as pen
import fs
from fs.osfs import OSFS
home = OSFS("c:")
# print(home.listdir("tom"))
directory = list(home.scandir('tom'))
#print(directory)
# for d in directory:
# print(d)
dir2 = home.filterdir('tom',files=['*.txt'], namespaces=['details'])
for d in dir2:
# 2016-03-19 13:33:00.656588+00:00
modified = pen.instance(d.modified).to_datetime_string()
print(d.name, d.size, modified)
#print(home.tree())
home.close()
class OdooAddonManager:
"""
Class wrapping the OAM behaviour
Attributes
----------
install_dir: OSFS
The installation directory
src_cache: dict
A dictionary containing, for each source type supporting cache (git), the temporary location of the previously
downloaded sources
odoo_version: str
Version of Odoo using the addons
desc_version: str
Version of the description file used to log changes
modules_to_install: dict
Modules to install as described in the YAML file
verbose_level: str
Level of details to print
"""
install_dir: OSFS
modules_to_install: Dict[str, Dict[str, Any]]
src_cache: Dict[str, Dict[str, Any]]
odoo_version: str
desc_version: str = None
verbose_level: str
_tmp_dir: TempFS = None
_hst_file: TextIOWrapper = None
_chglog_file: TextIOWrapper = None
def __init__(self,
description_file: str = None,
install_directory: str = ".",
verbose_level: str = VERBOSE_NONE):
self.install_dir = OSFS(install_directory)
self.verbose_level = verbose_level
self.src_cache = {
"git": {},
}
if description_file:
with open(description_file, "r") as description_file:
install_data = yaml.load(description_file, Loader=yaml.Loader)
self.modules_to_install = install_data.get("modules", [])
self.odoo_version = install_data.get("odoo_version")
self.desc_version = install_data.get("version")
def __del__(self):
self.install_dir.close()
if self._tmp_dir:
self._tmp_dir.close()
if self._hst_file:
self._hst_file.close()
if self._chglog_file:
self._chglog_file.close()
@property
def tmp_dir(self) -> TempFS:
"""
The temporary directory used to download modules before installing them if needed.
"""
if not self._tmp_dir:
self._tmp_dir = TempFS(TEMP_DIR_NAME)
return self._tmp_dir
@property
def history_file(self) -> TextIOWrapper:
"""
The history file where are logged the operations performed in the installation directory
"""
if not self._hst_file:
self._hst_file = open(
self.install_dir.getsyspath(HISTORY_FILE_NAME), 'a+')
return self._hst_file
@property
def changelog_file(self) -> TextIOWrapper:
"""
The markdown changelog file listing changes in a human-readable format
"""
if not self._chglog_file:
self._chglog_file = open(
self.install_dir.getsyspath(CHANGELOG_FILE_NAME), "a+")
self._chglog_file.seek(0)
if not self._chglog_file.read():
self._chglog_file.write("# CHANGELOG")
self._chglog_file.seek(0)
return self._chglog_file
def install_all(self, force: bool = False):
"""
Install all modules described in the description file.
:param force: whether to overwrite installed modules or not
"""
installed_modules = []
if self.verbose_level == VERBOSE_NONE:
with click.progressbar(self.modules_to_install) as modules:
for module in modules:
if self.install(module, force):
installed_modules.append(module)
else:
for module in self.modules_to_install:
if self.install(module, force):
installed_modules.append(module)
# Modules installed are removed from the list to avoid being processed twice e.g. in case of a refresh
for module in installed_modules:
self.modules_to_install.pop(module)
click.echo("{} module(s) installed.".format(len(installed_modules)))
def install(self, module_name: str, force: bool = False) -> bool:
"""
Install a single module from its source.
:param module_name: Name of the module
:param force: Whether to overwrite the module if it is already installed
:param: Whether the module has been installed or not
"""
success = False
self.pretty_print(module_name, "Installing...", level=VERBOSE_FULL)
source = self.modules_to_install[module_name]
origin_name = source.get("origin_name", module_name)
installed_version = self.get_module_version(module_name,
self.install_dir)
if force or not installed_version:
try:
source_fs = self.fetch_module_from_source(module_name)
self.install_from_fs(origin_name,
source_fs,
output_name=module_name)
version = self.get_module_version(module_name,
self.install_dir)
self.log(module_name,
OPERATION_INSTALL,
force=force,
extra=version)
if not force:
self.log_md(module_name,
OPERATION_INSTALL,
new_version=version)
success = True
except InvalidModuleError as err:
self.pretty_print(module_name,
err.message,
status=LOG_STATUS_ERROR,
level=VERBOSE_NONE)
except pygit2.errors.GitError:
self.pretty_print(
module_name,
"Installation failed - Could not fetch from Git repository.",
status=LOG_STATUS_ERROR,
level=VERBOSE_NONE)
except Exception as e:
self.pretty_print(module_name,
"Installation failed ({})".format(
type(e).__name__),
status=LOG_STATUS_ERROR,
level=VERBOSE_NONE)
else:
self.pretty_print(module_name,
"Already installed. Skipping installation.",
status=LOG_STATUS_WARNING,
level=VERBOSE_NORMAL)
return success
def update_all(self, force: bool = False):
"""
Update all modules
:param force: Whether to skip version check or not.
If True, modules are just replaced no matter if they are being downgraded or installed for the first time.
"""
updated_modules = []
if self.verbose_level == VERBOSE_NONE:
with click.progressbar(self.modules_to_install) as modules:
for module in modules:
if self.update(module, force):
updated_modules.append(module)
else:
for module in self.modules_to_install:
if self.update(module, force):
updated_modules.append(module)
# Modules updated are removed from the list to avoid being processed twice in case of a refresh
for module in updated_modules:
self.modules_to_install.pop(module)
click.echo("{} module(s) updated.".format(len(updated_modules)))
def update(self, module_name: str, force: bool = False) -> bool:
"""
Update a single module.
:param module_name: Name of the module
:param force: Whether to skip version check or not.
If True, modules are just replaced no matter if they are being downgraded or installed for the first time.
:return: Whether the module has been updated or not
"""
success = False
self.pretty_print(module_name, "Updating...", level=VERBOSE_FULL)
installed_version = self.get_module_version(module_name,
self.install_dir)
if force or installed_version:
try:
source_fs = self.fetch_module_from_source(module_name)
origin_name = self.modules_to_install[module_name].get(
"origin_name", module_name)
new_version = self.get_module_version(origin_name, source_fs)
if force or version.parse(new_version) >= version.parse(
installed_version):
self.pretty_print(module_name,
"Updating from {0} to {1}".format(
installed_version, new_version),
level=VERBOSE_FULL)
self.install_from_fs(origin_name,
source_fs,
output_name=module_name)
self.log(module_name,
OPERATION_UPDATE,
force=force,
extra="from {0} to {1}".format(
installed_version, new_version))
if not force:
self.log_md(module_name, OPERATION_UPDATE,
installed_version, new_version)
success = True
else:
self.pretty_print(
module_name,
"Fetched version ({0}) is inferior to current version ({1}). Skipping update."
.format(new_version, installed_version),
status=LOG_STATUS_ERROR,
level=VERBOSE_NORMAL)
except InvalidModuleError as err:
self.pretty_print(module_name,
err.message,
status=LOG_STATUS_ERROR,
level=VERBOSE_NONE)
except pygit2.errors.GitError:
self.pretty_print(
module_name,
"Update failed - Could not fetch from Git repository.",
status=LOG_STATUS_ERROR,
level=VERBOSE_NONE)
except Exception as e:
self.pretty_print(module_name,
"Update failed ({})".format(
type(e).__name__),
status=LOG_STATUS_ERROR,
level=VERBOSE_NONE)
else:
self.pretty_print(
module_name,
"Not installed. Skipping update.".format(module_name),
status=LOG_STATUS_WARNING,
level=VERBOSE_NORMAL)
return success
def uninstall_all(self, auto_confirm=False):
"""
Uninstall all modules that are installed but not present in the description file.
Ask confirmation to the user.
:param auto_confirm: Do not ask the user to confirm if True
"""
installed_modules = self.get_installed_modules()
modules_to_uninstall = set(installed_modules.keys()) - set(
self.modules_to_install.keys())
if not auto_confirm:
click.echo("The following modules will be removed:")
for module in modules_to_uninstall:
click.echo(module)
click.confirm('Do you want to continue?', abort=True)
count = 0
if self.verbose_level == VERBOSE_NONE:
with click.progressbar(modules_to_uninstall) as modules:
for module in modules:
count += self.uninstall(module)
else:
for module in modules_to_uninstall:
count += self.uninstall(module)
click.echo("{} module(s) removed.".format(count))
def uninstall(self, module_name: str) -> bool:
"""
Uninstall a single module if it is installed.
:param module_name: Name of the module
:return: Whether the module has been uninstalled or not
"""
success = False
if module_name in self.install_dir.listdir("."):
self.pretty_print(module_name,
"Uninstalling...",
level=VERBOSE_FULL)
self.install_dir.removetree(module_name)
success = True
self.log(module_name, OPERATION_UNINSTALL)
self.log_md(module_name, OPERATION_UNINSTALL)
self.pretty_print(module_name,
"Uninstalled.",
status=LOG_STATUS_OK,
level=VERBOSE_NORMAL)
else:
self.pretty_print(module_name,
"Not installed. Skipping uninstall.",
status=LOG_STATUS_ERROR,
level=VERBOSE_NORMAL)
return success
def get_installed_modules(self) -> Dict[str, str]:
"""
Scan installation directory to list currently installed modules
:return: A dictionary of module names as keys and their currently installed version as values
"""
modules = {}
for module in self.install_dir.scandir("."):
if module.is_dir and "__manifest__.py" in self.install_dir.listdir(
module.name):
manifest_file = self.install_dir.getsyspath(
join(module.name, "__manifest__.py"))
with open(manifest_file, "r") as manifest:
modules[module.name] = ast.literal_eval(
manifest.read())["version"]
return modules
@staticmethod
def get_module_version(module_name: str, directory: FS) -> str:
"""
Get the version of the module in the given directory
:param module_name: name of the module
:param directory: FS object pointing to the parent directory of the module
:return: version of the module or None if it is not present in the directory
"""
version = None
if module_name in directory.listdir("."):
manifest = directory.readtext(join(module_name, "__manifest__.py"))
version = ast.literal_eval(manifest)["version"]
return version
def fetch_module_from_source(self, module_name: str) -> FS:
"""
Download a module from its source if needed and return the directory where it is located.
:param module_name: Name of the module
:return: An FS object pointing to the module location
"""
source = self.modules_to_install[module_name]
source_fs: FS
if source["source_type"] == SOURCE_LOCAL_DIR:
source_fs = OSFS(source["path"])
elif source["source_type"] == SOURCE_LOCAL_ZIP:
source_fs = ZipFS(source["path"])
elif source["source_type"] == SOURCE_GIT:
source_fs = self.download_from_git(
module_name, source["url"],
source.get("branch", self.odoo_version),
source.get("path", "."))
return source_fs
def download_from_git(self,
module_name: str,
url: str,
branch: str,
path: str = ".") -> OSFS:
"""
Clone a git repository or find it in the source cache.
:param module_name: name of the module being installed
:param url: URL of the repository
:param branch: branch of the desired module version
:param path: path to the module inside the repository (default to '.')
:return: an OSFS object pointing to the module location inside the repository
"""
repo_dir_name = urlparse(url).path.replace("/", "_")
if url in self.src_cache["git"]:
self.pretty_print(module_name,
"Repository found in cache",
level=VERBOSE_FULL)
repo = self.src_cache["git"][url]
repo.checkout("refs/remotes/origin/{}".format(branch))
else:
self.pretty_print(module_name,
"Cloning repository",
level=VERBOSE_FULL)
repo = pygit2.clone_repository(
url,
self.tmp_dir.getsyspath(repo_dir_name),
checkout_branch=branch)
self.src_cache["git"][url] = repo
return OSFS(join(repo.workdir, path))
def install_from_fs(self,
name: str,
source_fs: FS,
path: str = ".",
output_name: str = None):
"""
Copy a module directory from where it is located to the installation directory.
:param name: Name of the module
:param source_fs: FS object pointing to the source location
:param path: Path to the module directory from the source location root
:param output_name: Name to give to the module's directory at installation
"""
path_to_module = join(path, name)
if name not in source_fs.listdir(path):
raise InvalidModuleError(
name,
"Module directory not found - Given path should be the parent directory"
)
if "__manifest__.py" not in source_fs.listdir(path_to_module):
raise InvalidModuleError(
name,
"Manifest not found - Given path should be the parent directory"
)
self.pretty_print(output_name,
"Copying from {}".format(
source_fs.desc(path_to_module)),
level=VERBOSE_FULL)
copy_dir(source_fs, path_to_module, self.install_dir, output_name
or name)
self.pretty_print(output_name,
"Installed and up to date.",
status=LOG_STATUS_OK,
level=VERBOSE_NORMAL)
def log(self,
module_name: str,
operation: str,
force=False,
extra: str = ""):
"""
Log an operation in the history file.
:param module_name: Name of the module
:param operation: Type of the operation
:param force: Whether the operation was performed with the force option or not
:param extra: Extra information to log
"""
log_line = "{0} - {1}{2}: {3} {4}\n".format(
datetime.now().replace(microsecond=0), operation,
" (forced)" if force else "", module_name, extra)
self.history_file.write(log_line)
def log_md(self,
module: str,
operation: str,
old_version: str = None,
new_version: str = None):
"""
Log an operation in the markdown log file in human-readable format.
:param module: Name of the module
:param operation: Type of the operation
:param old_version: Overwritten version of the module, in case of an update
:param new_version: New version of the module, in case of an installation/update
"""
current_log_content = self.changelog_file.read()
# Look for the section concerning the current version, or write a scaffold if not found
version = self.desc_version or datetime.today().strftime("%Y-%m-%d")
log_index = current_log_content.find("## {}".format(version))
if log_index >= 0:
new_log_content = current_log_content[log_index:]
else:
new_log_content = "\n\n## {}\n\n**Added**\n\n\n**Updated**\n\n\n**Removed**\n\n".format(
version)
log_index = len(current_log_content)
# Remove previous log entry concerning the module
if module in new_log_content:
new_log_content = re.sub(r"\n.*{}.*".format(module), "",
new_log_content)
# Append the new log line under the right operation type
if operation == OPERATION_INSTALL:
index = new_log_content.find("**Updated**") - 2
log_line = "\n * {0} ({1})".format(module, new_version)
elif operation == OPERATION_UPDATE:
index = new_log_content.find("**Removed**") - 2
log_line = "\n * {0} ({1} from {2})".format(
module, new_version, old_version)
elif operation == OPERATION_UNINSTALL:
index = len(new_log_content) - 1
log_line = "\n * {0}".format(module)
new_log_content = "{0}{1}{2}".format(new_log_content[:index], log_line,
new_log_content[index:])
# Overwrite file with the updated logs
old_log_content = current_log_content[:log_index]
self.changelog_file.truncate()
self.changelog_file.write(old_log_content + new_log_content)
def list_external_dependencies(self, raw=False, modules: List[str] = None):
"""
Show external dependencies of all installed modules.
:param raw: Whether to print only python dependencies in a 'requirements.txt' format
:param modules: If given, show dependencies of those modules only
"""
dependencies = self.get_all_dependencies(modules=modules)
if raw:
for dep in dependencies.get("python", []):
click.echo(dep)
else:
for type in dependencies:
click.echo(type)
for dep in dependencies[type]:
if type == "python":
dep_installed = self.check_python_dependency(dep)
click.echo("\t{0} {1}".format(
dep, "(OK)" if dep_installed else "(missing)"))
else:
click.echo("\t{}".format(dep))
def install_missing_dependencies(self, modules: List[str] = None):
"""
Install all missing dependencies.
:param modules: If given, install dependencies of those modules only
"""
dependencies = self.get_all_dependencies(modules=modules)
self.install_python_dependencies(dependencies.get("python", []))
def get_all_dependencies(self,
modules: List[str] = None
) -> Dict[str, List[str]]:
"""
Get all missing dependencies from the installed modules.
:param modules: If given, return dependencies of those modules only
:return: A dictionary containing a list of dependencies for each type
"""
# Filter installed modules to keep the ones given
modules = {mod: self.get_installed_modules()[mod] for mod in modules} if modules \
else self.get_installed_modules()
all_deps = {}
for module in modules:
module_deps = self.parse_dependencies(module, self.install_dir)
for type, deps in module_deps.items():
all_deps.setdefault(type, set()).update(set(deps))
return all_deps
@staticmethod
def parse_dependencies(module_name: str,
directory: FS) -> Dict[str, List[str]]:
"""
Retrieve external dependencies from a module's manifest.
:param module_name: Name of the module
:param directory: Location of the module
:return: A dictionary containing a list of dependencies for each type
"""
manifest = directory.readtext(join(module_name, "__manifest__.py"))
manifest_dict = ast.literal_eval(manifest)
return manifest_dict.get("external_dependencies", {})
@staticmethod
def check_python_dependency(dependency: str) -> bool:
"""
Check if a python dependency is satisfied i.e. if the python module is installed.
:param dependency: Name of the python module
:return: True if the module is installed, False otherwise
"""
try:
__import__(dependency)
except ImportError:
return False
return True
@staticmethod
def install_python_dependencies(dependencies: List[str]):
"""
Call pip to install the given python dependencies.
:param dependencies: List of python modules to install
"""
callable_pip.main("install", *dependencies)
def pretty_print(self,
module_name: str,
message: str = "",
status: str = LOG_STATUS_PENDING,
level: int = 0):
"""
Format and print a log to the console.
:param module_name: Name of the module concerned
:param message: Message to print
:param status: Status of the log ('pending', 'ok', 'warning', 'error')
:param level: Minimum verbose level to actually print the log (0, 1, 2)
"""
if level <= self.verbose_level:
if status == LOG_STATUS_OK:
msg_color = "green"
elif status == LOG_STATUS_WARNING:
msg_color = "yellow"
elif status == LOG_STATUS_ERROR:
msg_color = "red"
else:
msg_color = "white"
click.echo(
click.style(module_name.ljust(30), fg="blue") +
click.style(message, fg=msg_color))