def test_inform_critical(capsys):
with pytest.raises(SystemExit):
inform.critical('bla')
captured = capsys.readouterr()
assert captured[
0] == b' \x1b[1;91m\xe2\x9c\x98\x1b[0m\x1b[0;94m TEST_INFORM_CRITICAL \x1b[0mbla\n'.decode(
'utf8')
def create(self, dependencies):
"""
Create conda-environment with name self.name and passed dependencies.
Parameters
----------
dependencies: list
List of strings with dependencies (packagename with optional
version) to install inside environment.
Example:
['python=3.6', 'attrs=>17.3']
"""
deps = ' '.join([
f"'{_}'".replace(' >=',
'>=').replace(' <=',
'<=').replace(' ',
'=').replace('*', '')
for _ in dependencies
])
try:
utils.run_in_bash(
f'{CONDA_BIN} create -y -q -n {self.name} {deps}')
except CalledProcessError as err:
inform.error(f'Couldn\'t create environment {self.name}. '
'Following error occured:')
print(err.output.strip().decode('ascii'))
inform.error('Please check your meta.yaml-file and if '
'dependencies are available.')
inform.critical()
def connect_ssh(dst):
"""
Connect to destination host (dst).
Parameters
----------
dst: str
The destination-host as combination of "username@hostname".
Returns
-------
paramiko.SSHClient
The connection-object to the dst-host.
"""
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
username, hostname = dst.split('@')
try:
ssh.connect(hostname, 22, username, look_for_keys=True, compress=True)
except paramiko.ssh_exception.AuthenticationException:
inform.error('Authentication failed')
inform.critical()
except socket.gaierror:
inform.error('Name or service not known')
inform.critical()
else:
return ssh
def test(self, path=None):
"""
Runs all tests for the project in its tests folder with pytest.
Parameters
----------
path: pathlib.Path
The projects path.
Note
----
Calls :func:`Project.test` to update the projects conda-environment
before running the tests with pytest.
"""
if not path:
path = self.path
self.update()
inform.info('Running tests for project with pytest')
# using bash cause importing pytest in sublevels (testing pproject
# itself) can be pain in the ass
try:
res = utils.run_in_bash(
f'{Path(CONFIG["pproject_env"]) / "bin/pytest"} '
f'{" ".join(CONFIG["pytest_arguments"])} {path}')
print(res)
except CalledProcessError as err:
print(err.output.strip().decode('ascii'))
inform.critical()
inform.finished()
def build(self, publish=False, path=None):
"""
Builds a conda-package from the project.
First it runs all tests to ensure functionality of the resulting
package. Assuming the test-coverage is acceptable.
Then it checks for uncommited stuff inside the project and if it is
tagged with a version-tag. Finally the conda-package is build.
If the publish-flag is set it is uploaded to the conda-repo-server.
Else it is only built local.
Parameters
----------
publish: bool
Flag which indicates if the resulting conda-package should be
uploaded to the conda-repository-server defined in the config-file
or not.
path: pathlib.Path
The projects path.
Note
----
Runs :func:`Project.test` before build to check for failures.
Checks if uncommited stuff remains inside project-folder before build
with :func:`check_git_status`.
Also checks if a required git-tag exists with :func:`get_git_tag`.
To execute the conda-build command :func:`utils.run_in_bash` is used.
"""
if not path:
path = self.path
self.update_informations()
self.test(path=path)
checks = all([
self.git.status(),
self.git.get_tag(),
self.git.check_tag_on_remote()
])
if checks:
inform.info(f'Started build of {self.environment}')
pkg_name = conda.build_package(path=path,
pythonversion=self.pythonversion,
simulate=True)
inform.info(f'Returning packagename will be {pkg_name}')
try:
pkg_file = conda.build_package(
path=path, pythonversion=self.pythonversion)
bld_path = Path(CONFIG['conda_folder']) / 'conda-bld/linux-64'
pkg_path = str(bld_path / pkg_file)
inform.info(f'Built package is {localfilepath}')
if publish:
conda.publish_package_on_reposerver(pkg_path)
inform.finished()
except CalledProcessError:
inform.critical()
else:
inform.error('Git-status not clean.Check git status and tag.')
inform.critical()
def update_informations(self, create=False, path=None):
"""
Updates class-attributes with new informations collected from
meta.yaml, git config and current datetime.
Parameters
----------
create: bool
If project isn't created yet the name of the environment can't be
extracted from the meta.yaml. In this case it has to be combined
from companyname, namespace and projectname.
path: pathlib.Path
The projects-path.
Note
----
Uses :class:`conda.MetaYaml` to collect the name of environment
from the current project if **create** is False.
To collect git-specific informations :class:`git.GitRepo` is used.
Checks for valid project-definition with :class:`validators.SProject`.
"""
self.environment = f'{self.company}-{self.namespace}-{self.project}'
if create:
if not path:
path = self.path.parent
self.git = git.GitRepo(path=self.path)
else:
#if not self.environment in str(self.path):
# self.path = self.path / self.environment
if not path:
path = self.path
meta_yaml = conda.MetaYaml(path=path / CONFIG['meta_yaml_path'])
self.environment = meta_yaml.package_name
self.git = git.GitRepo(path=self.path)
self.version = self.git.get_tag()
now = dt.datetime.now()
self.year = now.strftime('%Y')
self.today = now.strftime('%Y-%m-%d %H:%M')
self.username = self.git.get_username()
self.email = self.git.get_email()
try:
validators.SProject(strict=True).load(self.__dict__)
except ValidationError as err:
inform.error('Can\'t collect project information.')
inform.error('Invalid value for following params:')
for key, value in err.messages.items():
inform.error(f'{key}: {value}')
inform.critical()
def __attrs_post_init__(self):
"""
Updates the class-attributes and validates the contents of the
meta.yaml-file.
"""
if not self.path:
self.path = Path.cwd() / CONFIG['meta_yaml_path']
if not self.path.exists():
raise AttributeError(f'Path {self.path} doesn\'t exist.')
self.update()
try:
validators.SMetaYaml(strict=True).load(self.get_content())
except ValidationError as err:
inform.error('meta.yaml has incorrect content.')
inform.error('Invalid value for following params:')
for key, value in err.messages.items():
inform.error(f'{key}: {value}')
inform.critical()
def create_remote(self, ssh, pythonversion, packagename, version,
projectpath):
"""
Release the package in its own conda-envrionment on a remote host.
Parameters
----------
ssh: paramiko.SSHClient
Connection via ssh where the conda-environment should be created.
pythonversion: str
pythonversion to create the environment for.
packagename: str
Name of the package to install inside the created environment.
version: str
Version of the package to install inside the created environment.
projectpath: str
Path of currrent project.
"""
inform.info('Creating env')
cmd_create = (f'{CONDA_BIN} create -y -q -n {self.name} '
f'python={pythonversion} '
f'{packagename}={version}')
_, stdout, stderr = ssh.exec_command(cmd_create)
stdout.channel.recv_exit_status()
err = stderr.read().strip().decode('ascii')
if err:
if 'CondaValueError: prefix already exists:' in err:
inform.info('Recreating env')
self.release_log(ssh, 'recreate', projectpath)
self.remove_remote(ssh, projectpath)
self.release_log(ssh, 'create', projectpath)
_, stdout, stderr = ssh.exec_command(cmd_create)
stdout.channel.recv_exit_status()
err = stderr.read().strip().decode('ascii')
else:
inform.error(f'Error during rollout ({cmd_create} => {err})')
inform.critical()
if err:
if not err.startswith('==> WARNING:'):
inform.error(
f'Error during rollout ({cmd_create} => {err})')
inform.critical()
else:
self.release_log(ssh, 'create', projectpath)
def remove(self):
"""
Remove conda-environment with name self.name if it exists.
"""
if self.exists():
try:
utils.run_in_bash(
f'{CONDA_BIN} env remove -q -y -n {self.name}')
except CalledProcessError as err:
err_message = err.output.strip().decode('ascii')
if 'CondaEnvironmentError:' in err_message:
inform.info('deactivating and retry')
utils.run_in_bash(
'source deactivate && '
f'{CONDA_BIN} env remove -q -y -n {self.name}')
else:
inform.error('Couldn\'t remove environment. '
'Following error occured:')
print(err_message)
inform.critical()
def run(options, path=Path.cwd()):
"""
Parameters
----------
options: argparse.Namespace
path: pathlib.Path
"""
if options.tool == 'info':
if options.infotype == 'general':
general_info()
elif options.infotype == 'project':
try:
meta_yaml = conda.MetaYaml()
company, namespace, projectname = (
meta_yaml.package_name.split('-'))
pythonversion = meta_yaml.pythonversion
prj = Project(company=company,
namespace=namespace,
project=projectname,
pythonversion=pythonversion,
path=path)
prj.update_informations(create=True, path=path)
prj.info()
except Exception as err:
print(err)
inform.error('Not a valid pproject-project!')
inform.critical()
else:
if options.tool not in ('create', ):
meta_yaml = conda.MetaYaml()
company, namespace, projectname = meta_yaml.package_name.split('-')
pythonversion = meta_yaml.pythonversion
else:
company, namespace, projectname = [
CONFIG['company'], options.namespace, options.projectname
]
pythonversion = options.pythonversion
prj = Project(company=company,
namespace=namespace,
project=projectname,
pythonversion=pythonversion,
path=path)
if options.tool == 'update':
prj.update_informations()
prj.update()
elif options.tool == 'test':
prj.update_informations()
prj.test(path=path)
elif options.tool == 'sphinx':
prj.update_informations()
prj.sphinx()
elif options.tool == 'create':
prj.create(on_vcs=options.remote, path=path)
elif options.tool == 'build':
prj.update_informations()
prj.build(publish=options.publish)
elif options.tool == 'version':
prj.update_informations()
prj.new_version(vtype=options.versiontype, message=options.message)
elif options.tool == 'release':
try:
envname = options.envname
except:
envname = f'{Path.cwd().name}_env'
prj.update_informations()
prj.release(dst=options.userathost, envname=envname)
def build_arguments(args):
"""
The argument parsing for the python-part of the pproject-tool.
Parameters
----------
args: list
Returns
-------
parsed_args: argparse.Namespace
"""
# TODO: make dynamic with CONFIG['use_vcs']
try:
assert all([
_ in (list(string.ascii_letters + string.digits) + ['_'])
for _ in list(CONFIG['company'])
])
except AssertionError:
inform.error(
'Your company-name contains unsupported chars (only letters and "_" are allowed)'
)
inform.critical()
vcs = CONFIG['vcs']['use']
if vcs == 'gitlab' and VCS_SETTINGS['use_groups']:
try:
available_namespaces = [
grp.split('-')[1] for grp in git.get_gitlab_groups().keys()
if '-' in grp and grp.split('-')[0] == CONFIG['company']
]
if not available_namespaces:
raise AttributeError
except:
available_namespaces = OFFLINE_NAMESPACES
else:
available_namespaces = OFFLINE_NAMESPACES
parser = argparse.ArgumentParser(description='ouroboros-tools-pproject')
tools = parser.add_subparsers(
description='pproject supports different tools. These are:')
create = tools.add_parser(
'create', description='creates new pproject supported project')
create.set_defaults(tool='create')
create.add_argument('-r', '--remote', action='store_true', default=False)
namespaces = create.add_subparsers(
description='the following namespaces are supported')
#for namespace_name in ('services', 'products', 'modules', 'operations'):
for namespace_name in available_namespaces:
namespace = namespaces.add_parser(
namespace_name,
description=f'Uses "{namespace_name}" as namespace')
namespace.set_defaults(namespace=namespace_name)
namespace.add_argument('-n', '--projectname', type=str, required=True)
namespace.add_argument('-p',
'--pythonversion',
type=str,
default='3.6')
build = tools.add_parser('build')
build.set_defaults(tool='build')
build.add_argument('-p', '--publish', action='store_true', default=False)
for tool in ('test', 'sphinx', 'update'):
new_tool = tools.add_parser(tool)
new_tool.set_defaults(tool=tool)
info_parser = tools.add_parser('info')
info_parser.set_defaults(tool='info')
infotypes = info_parser.add_subparsers()
info_general = infotypes.add_parser('general')
info_general.set_defaults(infotype='general')
info_project = infotypes.add_parser('project')
info_project.set_defaults(infotype='project')
new_version = tools.add_parser('version')
new_version.add_argument('-m', '--message', type=str, required=True)
new_version.set_defaults(tool='version')
versiontypes = new_version.add_subparsers()
for versiontype in ('major', 'minor', 'patch'):
vtype = versiontypes.add_parser(versiontype)
vtype.set_defaults(versiontype=versiontype)
release = tools.add_parser('release')
release.set_defaults(tool='release')
release.add_argument('-d', '--userathost', type=str)
release.add_argument('-e', '--envname', type=str)
return parser.parse_args(args)
def new_version(self,
vtype,
message='New version triggered by pproject',
path=None):
"""
Raises the passed vtype (versiontype) inside version (git-tag) of the
project.
Version has to be in semantic versioning style (see https://semver.org/
for details).
Example for version in semantic versioning style:
General: "major.minor.patch"
Example: "1.0.1"
Parameters
----------
vtype: str
The part to raise inside current version. Valid are: "major",
"minor" and "patch"
message: str
The message to use for the git tag.
path_: pathlib.Path()
The projects path.
Note
----
First checks if there is uncommited stuff inside the current branch.
If there is uncommited stuff, the operation is aborted.
Else checks if the gitlab is reachable.
Then the current version is collected.
The passed **vtype** is raised by 1. Finally the resulting version-tag
is added with passed **message** and the new tag is pushed to gitlab.
For these operations :class:`git.GitRepo` is used.
"""
if not path:
path = self.path
assert all([isinstance(vtype, str), isinstance(message, str)])
assert vtype in ('major', 'minor', 'patch')
self.update_informations(path=path)
if self.git.status():
if not git.check_remote_vcs():
inform.error('Remote vcs not accessable')
inform.critical()
else:
self.version = self.git.get_tag()
inform.info(f'Current version is {self.version}')
major, minor, patch = [
int(_) for _ in self.version.split('-')[0].split('.')
]
vrs = {'major': major, 'minor': minor, 'patch': patch}
vrs[vtype] += 1
if vtype != 'patch':
vrs['patch'] = 0
if vtype == 'major':
vrs['minor'] = 0
res_version = f'{vrs["major"]}.{vrs["minor"]}.{vrs["patch"]}'
inform.info(f'Resulting version {res_version}')
try:
self.git.create_tag(res_version, message)
self.git.push_tag(res_version)
except CalledProcessError:
inform.error('Can\'t push to remote vcs.')
inform.critical()
self.version = res_version
else:
inform.error(
'No new git-tag possible, uncommited stuff in project')
inform.critical()
def create(self, on_vcs=False, path=None):
"""
Creates new project based on a defined skeleton either local or on
gitlab with cookiecutter, creates the base-conda-environment for
developing the new project and manages corresponding git-tasks.
Parameters
----------
on_vcs: bool
Flag to define if project should be added to remote vcs after
creation.
path: pathlib.Path
The projects path.
Note
----
Calls :func:`Project.update_informations` with **create=True** to
update the **attributes** of the project.
For git operations the function :func:`utils.run_in_bash` is called.
After creation :func:`Project.update` with **path** set to current
working directory is called to create the conda-environment of the
project.
"""
if not path:
path = self.path.parent
self.update_informations(create=True, path=path)
assert isinstance(path, Path)
if not (path / self.environment).exists():
if on_vcs:
if not git.check_remote_vcs():
inform.error('Remote vcs not accessable')
inform.critical()
inform.info(f'Creating project {self.environment}')
cookiecutter(CONFIG['skeleton_repo'],
checkout=str(self.pythonversion),
output_dir=str(Path.cwd()),
no_input=True,
extra_context=self.__dict__)
inform.info('Created folder')
os.chdir(str((path / self.environment).absolute()))
if not self.environment in str(self.path):
self.path = self.path / self.environment
self.git = git.GitRepo(path=self.path)
if on_vcs:
create_on_remote_res = git.create_on_remote_vcs(
company=self.company,
namespace=self.namespace,
project=self.project,
username=self.git.get_username())
inform.info('Initializing git')
self.git.initialize()
inform.info('Adding files')
self.git.add_all()
inform.info('Commiting')
self.git.commit()
vcs_ssh = VCS_SETTINGS['ssh']
vcs = CONFIG['vcs']['use']
vcs_use_groups = VCS_SETTINGS['use_groups']
if on_vcs:
if vcs == 'gitlab' and vcs_use_groups:
git_repo = f'{create_on_remote_res}/{self.project}.git'
else:
git_repo = f'{create_on_remote_res}.git'
if ':' in vcs_ssh:
git_origin = f'{vcs_ssh}/{git_repo}'
else:
git_origin = f'{vcs_ssh}:{git_repo}'
inform.info(f'Setting origin to {git_origin}')
self.git.set_origin(git_origin)
inform.info(f'Pushing to origin')
self.git.push_to_branch('master')
inform.finished()
self.update(path=(path / self.environment).absolute())
else:
inform.error('Folder already exists')
inform.critical()