def _recreate_repository(self, path: str, repo: dict):
""" Recreate (remove and clone) repository on specifed path.
:param path: string path to repository
:type path: string
:param repo: dict representing item from deployment file
:type repo: dict
:raises: :class:`DeployException`
"""
import git
from pypeapp.lib.git_progress import _GitProgress
try:
shutil.rmtree(path)
except (OSError, PermissionError) as e:
raise DeployException(("Cannot remove existing non"
" git repository.{}".format(e))) from e
else:
# clone repo
try:
git.Repo.clone_from(repo.get('url'),
path,
progress=_GitProgress(),
env=None,
b=repo.get('branch') or repo.get('tag'),
recursive=True)
except git.exc.GitCommandError as e:
raise DeployException("Git clone failed for {}".format(
repo.get("url"))) from e
def deploy(self, force=False):
""" Do repositories deployment and install python dependencies.
Go throught deployment file and install repositories specified
there. Also add additional python dependencies with pip.
:param force: overwrite existng repos if it's working tree is
dirty.
:type force: bool
:raises: :class:`DeployException`
"""
import git
from pypeapp.lib.git_progress import _GitProgress
settings = self._determine_deployment_file()
deploy = self._read_deployment_file(settings)
term = Terminal()
# go throught repositories
term.echo(">>> Deploying repositories ...")
for ritem in deploy.get('repositories'):
path = os.path.join(self._pype_root, "repos", ritem.get('name'))
term.echo(" -- processing [ {} / {} ]".format(
ritem.get('name'),
ritem.get('branch') or ritem.get('tag')))
if self._validate_is_directory(path):
# path exists
# is it repository?
if not self._validate_is_repo(path):
# no, remove dir no matter of content
term.echo(" - removing existing directory and cloning...")
self._recreate_repository(path, ritem)
else:
# dir is repository
repo = git.Repo(path)
# is it right one?
if not self._validate_origin(path, str(ritem.get('url'))):
# repository has different origin then specified
term.echo("!!! repository has different origin. ")
if (self._validate_is_dirty(path) is True
and force is False):
raise DeployException(('Invalid repository on '
'path {}'.format(path)))
term.echo(" -> recreating repository. ")
self._recreate_repository(path, ritem)
pass
if (self._validate_is_dirty(path) is True
and force is False):
raise DeployException(("Repo on path [ {} ] has dirty"
" worktree").format(path), 300)
# are we on correct branch?
if not ritem.get('tag'):
if not self._validate_is_branch(
path, ritem.get('branch')):
term.echo(" . switching to [ {} ] ...".format(
ritem.get('branch')))
branch = repo.create_head(ritem.get('branch'),
'HEAD')
branch.checkout(force=force)
# update repo
term.echo(" . updating ...")
repo.remotes.origin.fetch(tags=True, force=True)
# build refspec
if ritem.get('branch'):
refspec = "refs/heads/{}".format(ritem.get('branch'))
repo.remotes.origin.pull(refspec)
elif ritem.get('tag'):
tags = repo.tags
if ritem.get('tag') not in tags:
raise DeployException(
("Tag {} is missing on remote "
"origin").format(ritem.get('tag')))
t = tags[ritem.get('tag')]
term.echo(" . tag: [{}, {} / {}]".format(
t.name, t.commit, t.commit.committed_date))
repo.remotes.origin.pull(ritem.get('tag'))
else:
# path doesn't exist, clone
try:
git.Repo.clone_from(ritem.get('url'),
path,
progress=_GitProgress(),
env=None,
b=ritem.get('branch')
or ritem.get('tag'),
recursive=True)
except git.exc.GitCommandError as e:
raise DeployException("Git clone failed for {}".format(
ritem.get("url"))) from e
# Go through zip files.
term.echo(">>> Deploying archive files ...")
if deploy.get('archive_files'):
for item in deploy.get("archive_files"):
term.echo(" -- processing [ {} ]".format(
item.get("extract_path")))
path = os.path.normpath(
os.path.join(self._pype_root, item.get("extract_path")))
if self._validate_is_directory(path):
term.echo(" - removing existing directory.")
shutil.rmtree(path)
# Download archive file.
archive_type = item.get('archive_type')
basename = os.path.split(path)[-1]
filename = '.'.join([basename, archive_type])
archive_file_path = tempfile.mkdtemp(basename + '_archive')
archive_file_path = os.path.join(archive_file_path, filename)
if item.get("vendor"):
source = os.path.join(os.environ.get("PYPE_ROOT"),
'vendor', 'packages',
item.get("vendor"))
if not os.path.isfile(source):
raise DeployException(
"Local archive {} doesn't exist".format(source))
shutil.copyfile(source, archive_file_path)
if item.get("url"):
term.echo(" - downloading [ {} ]".format(item.get("url")))
success = self._download_file(item.get("url"),
archive_file_path)
if not success:
raise DeployException(
"Failed to download [ {} ]".format(
item.get("url")),
130 # noqa: E501
)
# checksum
if item.get('md5_url'):
response = urlopen(item.get('md5_url'))
md5 = response.read().decode('ascii').split(" ")[0]
calc = self.calculate_checksum(archive_file_path)
if md5 != calc:
raise DeployException(
"Checksum failed {} != {} on {}".format(
md5, calc, archive_file_path))
# Extract files from archive
if archive_type in ['zip']:
zip_file = zipfile.ZipFile(archive_file_path)
zip_file.extractall(path)
elif archive_type in [
'tar', 'tgz', 'tar.gz', 'tar.xz', 'tar.bz2'
]:
if archive_type == 'tar':
tar_type = 'r:'
elif archive_type.endswith('xz'):
tar_type = 'r:xz'
elif archive_type.endswith('gz'):
tar_type = 'r:gz'
elif archive_type.endswith('bz2'):
tar_type = 'r:bz2'
else:
tar_type = 'r:*'
try:
tar_file = tarfile.open(archive_file_path, tar_type)
except tarfile.ReadError:
raise DeployException(
"corrupted archive: also consider to download the "
"archive manually, add its path to the url, run "
"`./pype deploy`")
tar_file.extractall(path)
tar_file.close()
# Move folders/files if skip first subfolder is set
if item.get('skip_first_subfolder', False):
self.move_subfolders_to_main(path)
# install python dependencies
term.echo(">>> Adding python dependencies ...")
for pitem in deploy.get('pip'):
term.echo(" -- processing [ {} ]".format(pitem))
try:
subprocess.check_output(
[sys.executable, '-m', 'pip', 'install', pitem])
except subprocess.CalledProcessError as e:
raise DeployException('PIP command failed with {}'.format(
e.returncode)) from e