def test_clone_or_pull_repository_by_updating_outdated_repository(tmpdir):
modules_directory = Path(tmpdir.mkdir('modules'))
repo_dir = clone_repo(
user='******',
repository='color-schemes.astrality',
modules_directory=modules_directory,
)
# Move master to first commit in repository
result = run_shell(
command='git reset --hard 2b8941a',
timeout=5,
fallback=False,
working_directory=repo_dir,
)
assert result is not False
# The readme does not exist in this commit
readme = repo_dir / 'README.rst'
assert not readme.is_file()
# The following pull should reintroduce the README into the directory
updated_repo_dir = clone_or_pull_repo(
user='******',
repository='color-schemes.astrality',
modules_directory=modules_directory,
)
assert updated_repo_dir == repo_dir
assert readme.is_file()
def execute( # type: ignore
self,
default_timeout: Union[int, float] = 0,
dry_run: bool = False,
) -> Optional[Tuple[str, str]]:
"""
Execute shell command action.
:param default_timeout: Run timeout in seconds if no specific value is
specified in `options`.
:param dry_run: If True, skip and log commands to be executed.
:return: 2-tuple containing the executed command and its resulting
stdout.
"""
if self.null_object:
# Null objects do nothing
return None
command = self.option(key='shell')
timeout = self.option(key='timeout')
logger = logging.getLogger(__name__)
if dry_run:
logger.info(
f'SKIPPED: [run] Command: "{command}" (timeout={timeout}).',
)
return command, ''
logger.info(f'Running command "{command}".')
result = utils.run_shell(
command=command,
timeout=timeout or default_timeout,
working_directory=self.directory,
)
return command, result
def execute(
self,
default_timeout: Union[int, float] = 0,
) -> Optional[Tuple[str, str]]:
"""
Execute shell command action.
:param default_timeout: Run timeout in seconds if no specific value is
specified in `options`.
:return: 2-tuple containing the executed command and its resulting
stdout.
"""
if self.null_object:
# Null objects do nothing
return None
command = self.option(key='shell')
timeout = self.option(key='timeout')
logger = logging.getLogger(__name__)
logger.info(f'Running command "{command}".')
result = utils.run_shell(
command=command,
timeout=timeout or default_timeout,
working_directory=self.directory,
)
return command, result
def execute(self, dry_run: bool = False) -> Dict[Path, Path]:
"""
Copy from `content` path to `target` path.
:param dry_run: If True, skip and log copy creation(s).
:return: Dictionary with content keys and copy values.
"""
if self.null_object:
return {}
content = self.option(key='content', path=True)
target = self.option(key='target', path=True)
include = self.option(key='include', default=r'(.+)')
permissions = self.option(key='permissions', default=None)
copies = utils.resolve_targets(
content=content,
target=target,
include=include,
)
logger = logging.getLogger(__name__)
for content, copy in copies.items():
self.copied_files[content].add(copy)
log_msg = f'[copy] Content: "{content}" -> Target: "{target}".'
if dry_run:
logger.info('SKIPPED: ' + log_msg)
continue
logger.info(log_msg)
self.creation_store.mkdir(copy.parent)
self.creation_store.backup(path=copy)
utils.copy(
source=content,
destination=copy,
follow_symlinks=False,
)
self.creation_store.insert_creation(
content=content,
target=copy,
method=persistence.CreationMethod.COPY,
)
if permissions and not dry_run:
for copy in copies.values():
result = utils.run_shell(
command=f'chmod {permissions} "{copy}"',
timeout=1,
fallback=False,
)
if result is False:
logger = logging.getLogger(__name__)
logger.error(
f'Could not set "{permissions}" '
f'permissions for copy "{target}"',
)
return copies
def clone_or_pull_repo(
user: str,
repository: str,
modules_directory: Path,
timeout: Union[int, float] = 50,
) -> Path:
"""
Fetch newest version of GitHub repository.
:param user: GitHub username.
:param repository: Repository name.
:param modules_directory: Directory containing cloned repositories.
:param timeout: Time to wait for successful clone.
:return: Path to cloned repository.
"""
github_repo_directory = modules_directory / user / repository
if not github_repo_directory.is_dir():
# The repository does not exist, so we clone it
return clone_repo(
user=user,
repository=repository,
modules_directory=modules_directory,
timeout=timeout,
)
logger = logging.getLogger(__name__)
if not (github_repo_directory / '.git').is_dir():
logger.error(
f'Tried to update git module directory "{github_repo_directory}", '
'but the directory does not contain a ".git" sub-directory.', )
return github_repo_directory
result = run_shell(
command='GIT_TERMINAL_PROMPT=0 git pull',
timeout=timeout,
fallback=False,
working_directory=github_repo_directory,
allow_error_codes=True,
)
if result is False:
raise GithubModuleError(
f'Could not git pull module directory "{github_repo_directory}". '
f'Return value from git pull operation: "{result}".', )
return github_repo_directory
def test_use_of_autoupdating_github_source(
self,
patch_xdg_directory_standard,
):
"""When autoupdate is True, the latest revision should be pulled."""
github_module_source = GithubModuleSource(
enabling_statement={
'name': 'github::jakobgm/test-module.astrality',
'autoupdate': True,
},
modules_directory=Path('/what/ever'),
)
# The repository is lazely cloned, so we need to get the config
github_module_source.modules({})
repo_dir = (patch_xdg_directory_standard /
'repositories/github/jakobgm/test-module.astrality')
assert repo_dir.is_dir()
# Move master to first commit in repository
result = run_shell(
command='git reset --hard d4c9723',
timeout=5,
fallback=False,
working_directory=repo_dir,
)
assert result is not False
# The readme does not exist in this commit
readme = repo_dir / 'README.rst'
assert not readme.is_file()
del github_module_source
github_module_source = GithubModuleSource(
enabling_statement={
'name': 'github::jakobgm/test-module.astrality',
'autoupdate': True,
},
modules_directory=Path('/what/ever'),
)
github_module_source.modules({})
# The autoupdating should update the module to origin/master
# containing the README.rst file
assert readme.is_file()
def clone_repo(
user: str,
repository: str,
modules_directory: Path,
timeout: Union[int, float] = 50,
) -> Path:
"""
Clone Github `user`/`repository` to modules_directory.
The resulting repository is placed in:
<modules_directory>/<user>/<repository>.
:param user: GitHub username.
:param repository: Repository name.
:param modules_directory: Directory containing cloned repositories.
:param timeout: Time to wait for successful clone.
:return: Path to cloned repository.
"""
github_user_directory = modules_directory / user
github_user_directory.mkdir(parents=True, exist_ok=True)
repository_directory = github_user_directory / repository
github_url = \
f'https://github.com/{user}/{repository}.git {repository_directory}'
# Fail on git credential prompt: https://serverfault.com/a/665959
result = run_shell(
command='GIT_TERMINAL_PROMPT=0 git clone ' + github_url,
timeout=timeout,
fallback=False,
working_directory=github_user_directory,
allow_error_codes=True,
)
if not repository_directory.is_dir() or result is False:
try:
github_user_directory.rmdir()
except OSError:
pass
raise GithubModuleError(
f'Could not clone repository "{user}/{repository}.\n'
f'Return value from cloning operation: "{result}".', )
return repository_directory
def __init__(
self,
requirements: RequirementDict,
directory: Path,
timeout: Union[int, float] = 1,
) -> None:
"""Construct RequirementStatement object."""
self.successful: bool = True
self.repr: str = ''
# Check shell requirements
if 'shell' in requirements:
command = requirements['shell']
result = utils.run_shell(
command=command,
fallback=False,
timeout=requirements.get('timeout') or timeout,
working_directory=directory,
)
if result is False:
self.repr = f'Unsuccessful command: "{command}", '
self.successful = False
else:
self.repr = f'Sucessful command: "{command}" (OK), '
# Check environment requirements
if 'env' in requirements:
env_variable = requirements['env']
if env_variable not in os.environ:
self.repr += f'Missing environment variable: "{env_variable}", '
self.successful = False
else:
self.repr += 'Found environment variable: ' \
f'"{env_variable}" (OK), '
# Check installed requirements
if 'installed' in requirements:
program = requirements['installed']
in_path = bool(shutil.which(program))
if not in_path:
self.repr += f'Program not installed: "{program}", '
self.successful = False
else:
self.repr += f'Program installed: "{program}" (OK), '
def test_use_of_autoupdating_github_source(self, tmpdir):
modules_directory = Path(tmpdir)
github_module_source = GithubModuleSource(
enabling_statement={
'name': 'github::jakobgm/test-module.astrality',
'autoupdate': True,
},
modules_directory=modules_directory,
)
# The repository is lazely cloned, so we need to get the config
github_module_source.modules({})
repo_dir = modules_directory / 'jakobgm' / 'test-module.astrality'
assert repo_dir.is_dir()
# Move master to first commit in repository
result = run_shell(
command='git reset --hard d4c9723',
timeout=5,
fallback=False,
working_directory=repo_dir,
)
assert result is not False
# The readme does not exist in this commit
readme = repo_dir / 'README.rst'
assert not readme.is_file()
del github_module_source
github_module_source = GithubModuleSource(
enabling_statement={
'name': 'github::jakobgm/test-module.astrality',
'autoupdate': True,
},
modules_directory=modules_directory,
)
github_module_source.modules({})
# The autoupdating should update the module to origin/master
# containing the README.rst file
assert readme.is_file()
def compile_template(
template: Path,
target: Path,
context: Context,
shell_command_working_directory: Path,
permissions: Optional[str] = None,
) -> None:
"""
Compile template to target destination with specific context.
If `permissions` is provided, the target file will have its file mode set
accordingly.
permissions='755' -> chmod 755
permissions='u+x' -> chmod u+x
"""
logger.info(f'[Compiling] Template: "{template}" -> Target: "{target}"')
result = compile_template_to_string(
template=template,
context=context,
shell_command_working_directory=shell_command_working_directory,
)
# Create parent directories if they do not exist
os.makedirs(target.parent, exist_ok=True)
with open(target, 'w') as target_file:
target_file.write(result)
# Copy template's file permissions to compiled target file
shutil.copymode(template, target)
if permissions:
result = utils.run_shell(
command=f'chmod {permissions} {target}',
timeout=0,
fallback=False,
)
if result is False:
logger.error(
f'Could not set "{permissions}" permissions for "{target}"', )
