def test_build_with_alternate_config(self):
pkgname = 'new_pkg'
alt_settings = 'alt-settings.json'
with temp_chdir() as d:
runner = CliRunner()
result = runner.invoke(kecpkg, ['new', pkgname, '--no-venv'])
package_dir = get_package_dir(pkgname)
self.assertTrue(os.path.exists(package_dir))
# set alternative settings path
settings = copy_default_settings()
settings["package_name"] = pkgname
save_settings(settings,
package_dir=package_dir,
settings_filename=alt_settings)
os.chdir(package_dir)
result = runner.invoke(
kecpkg, ['build', pkgname, '--config', alt_settings])
self.assertEqual(
result.exit_code, 0,
"Results of the run were: \n---\n{}\n---".format(
result.output))
self.assertExists(os.path.join(package_dir, 'dist'))
dist_dir_contents = os.listdir(os.path.join(package_dir, 'dist'))
self.assertTrue(len(dist_dir_contents), 1)
self.assertTrue(
pkgname in dist_dir_contents[0],
"the name of the pkg `{}` should be in the name of "
"the built kecpkg `{}`".format(pkgname, dist_dir_contents[0]))
def test_build_with_extra_ignores(self):
pkgname = 'new_pkg'
with temp_chdir() as d:
runner = CliRunner()
result = runner.invoke(kecpkg, ['new', pkgname, '--no-venv'])
package_dir = get_package_dir(pkgname)
self.assertTrue(os.path.exists(package_dir))
ensure_dir_exists(os.path.join(package_dir, 'data'))
# add additional files (to exclude for building later)
touch_file(os.path.join(package_dir, 'data', 'somefile.txt'))
touch_file(
os.path.join(package_dir, 'local_extra_file.someext.txt'))
os.chdir(package_dir)
# check if those files exists
package_dir_contents = os.listdir(os.path.join(package_dir))
self.assertTrue(
'local_extra_file.someext.txt' in package_dir_contents)
self.assertTrue('data' in package_dir_contents)
# set exclude_paths in settings
settings = copy_default_settings()
settings["exclude_paths"] = ["data", "local_extra_file.*"]
save_settings(settings, package_dir=package_dir)
# run the builder
result = runner.invoke(kecpkg, ['build', pkgname, '--verbose'])
self.assertEqual(
result.exit_code, 0,
"Results of the run were: \n---\n{}\n---".format(
result.output))
self.assertExists(os.path.join(package_dir, 'dist'))
# check the zip such that the extra files are not packaged
dist_list = os.listdir(os.path.join(package_dir, 'dist'))
zipfile = ZipFile(os.path.join(package_dir, 'dist', dist_list[0]),
'r')
contents = zipfile.namelist()
self.assertFalse('local_extra_file.someext.txt' in contents)
self.assertFalse('data' in contents)
def new(package=None, **options):
"""
Create a new package directory structure.
<pkg dir>
+-- venv
| +-- ... <the virtualenvironment>
+-- README.md
+-- requirements.txt
+-- script.py
+-- package-info.json
+-- .gitignore
+-- .kecpkg-settings.json
"""
settings = load_settings(lazy=True, settings_filename=options.get('settings_filename'))
if not settings:
settings = copy_default_settings()
package_root_dir = os.getcwd()
# set the package name, clean an normalise using snake_case
package_name = package or click.prompt("Package name")
package_name = normalise_name(package_name)
# save to settings
settings['package_name'] = package_name
package_dir = os.path.join(package_root_dir, package_name)
if os.path.exists(package_dir):
echo_failure("Directory '{}' already exists.".format(package_dir))
sys.exit(1)
if not package:
settings['version'] = click.prompt('Version', default=settings.get('version', '0.0.1'))
settings['description'] = click.prompt('Description', default='')
settings['name'] = click.prompt('Author', default=settings.get('name', os.environ.get('USER', '')))
settings['email'] = click.prompt('Author\'s email', default=settings.get('email', ''))
settings['python_version'] = click.prompt('Python version (choose from: {})'.format(settings.get('pyversions')),
default='3.5')
settings['exclude_paths'] = click.prompt("Exclude additional paths from kecpkg (eg. 'data, input')",
default=settings.get('exclude_paths', ''),
value_proc=process_additional_exclude_paths)
if options.get('script'):
script_base = normalise_name(options.get('script').replace('.py', ''))
echo_info('Setting the script to `{}`'.format(script_base))
settings['entrypoint_script'] = script_base
if options.get('venv'):
settings['venv_dir'] = normalise_name(options.get('venv'))
echo_info("Creating package structure")
create_package(package_dir, settings=settings)
if not options.get('no_venv'):
echo_info("Creating virtual environment")
create_venv(package_dir, settings, pypath=None, use_global=options.get('global_packages'),
verbose=options.get('verbose'))
pip_install_venv(package_dir, settings, verbose=options.get('verbose'))
else:
settings['venv_dir'] = None
# save the settings (in the package_dir)
save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename'))
echo_success('Package `{package_name}` created in `{package_dir}`'.format(package_name=package_name,
package_dir=package_dir))
def config(package, **options):
"""Manage the configuration (or settings) of the package.
The various settings in the .kecpkg-settings.json file are:
package_name: name of the package
version: version number of the package
description: longer description of the package
name: name of the author of the package
email: email of the author of the package
python_version: python version on which the kecpkg will run, corresponds
with an executable KE-crunch environment
entrypoint_script: the name of the script that will be executed first
entrypoint_func: function name inside the script that will be executed.
Ensure that it takes *args, **kwargs.
venv_dir: python virtual environment directory in the development environment
requirements_filename: name of the requirements file with list of package that
will be installed before running
build_dir: directory where the built kecpkg will be stored
exclude_paths: list of paths that will be excluded from the package, next to
the build in excludes
url: url where the package will be uploaded
token: token of the user under which the package is uploaded
scope_id: identification of the scope under which the package is uploaded
service_id: identification under which the package is re-uploaded
(or recently uploaded)
last_upload: date and time of the last upload
"""
echo_info('Locating package ``'.format(package))
package_dir = get_package_dir(package_name=package)
package_name = os.path.basename(package_dir)
echo_info('Package `{}` has been selected'.format(package_name))
if options.get('init'):
if os.path.exists(os.path.join(package_dir, options.get('settings_filename'))) and \
click.confirm('Are you sure you want to overwrite the current settingsfile '
'(old settings will be a backup)?'):
copy_path(
os.path.join(package_dir, options.get('settings_filename')),
os.path.join(
package_dir,
"{}-backup".format(options.get('settings_filename'))))
echo_info('Creating new settingsfile')
settings = copy_default_settings()
settings['package_name'] = package_name
save_settings(settings,
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
settings = load_settings(package_dir=package_dir)
if options.get('interactive'):
settings['version'] = click.prompt('Version',
default=settings.get(
'version', '0.0.1'))
settings['description'] = click.prompt('Description',
default=settings.get(
'description', ''))
settings['name'] = click.prompt('Author',
default=settings.get(
'name', os.environ.get('USER',
'')))
settings['email'] = click.prompt('Author\'s email',
default=settings.get('email', ''))
settings['python_version'] = click.prompt(
'Python version (choose from: {})'.format(
settings.get('pyversions')),
default='3.5')
settings['exclude_paths'] = click.prompt(
"Exclude additional paths from kecpkg (eg. 'data, input')",
default=settings.get('exclude_paths', ''),
value_proc=process_additional_exclude_paths)
save_settings(settings,
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
if options.get('set_key'):
k, v = options.get('set_key')
if options.get('verbose'):
echo_info("Set the key '{}' to value '{}'".format(k, v))
settings[k] = v
save_settings(settings,
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
if options.get('get_key'):
echo_info(
tabulate([
(options.get('get_key'), settings.get(options.get('get_key')))
],
headers=("key", "value")))
return
if options.get('verbose'):
echo_info(tabulate(settings.items(), headers=("key", "value")))
if not options.get('interactive'):
echo_success('Settings file identified and correct')
def upload(package=None,
url=None,
username=None,
password=None,
token=None,
scope=None,
scope_id=None,
kecpkg=None,
**options):
"""
Upload built kecpkg to KE-chain.
If no options are provided, the interactive mode is triggered.
"""
package_name = package or get_package_name() or click.prompt(
'Package name')
settings = load_settings(
package_dir=get_package_dir(package_name),
settings_filename=options.get('settings_filename'))
if not url or not ((username and password) or token):
url = click.prompt('Url (incl http(s)://)',
default=settings.get('url') or url)
username = click.prompt('Username',
default=settings.get('username') or username)
password = click.prompt('Password', hide_input=True)
# set the interactive world to True for continuation sake
options['interactive'] = True
elif not options.get('interactive'):
url = url or settings.get('url')
username = username or settings.get('username')
token = token or settings.get('token')
scope_id = scope_id or settings.get('scope_id')
client = Client(url)
client.login(username=username, password=password, token=token)
# scope finder
if not scope_id and settings.get('scope_id') and \
click.confirm("Do you wish to use the stored `scope_id` in settings: `{}`".format(
settings.get('scope_id')), default=True):
scope_id = settings.get('scope_id')
if not scope_id:
scopes = client.scopes()
scope_matcher = [
dict(number=i, scope_id=scope.id, scope=scope.name)
for i, scope in zip(range(1, len(scopes)), scopes)
]
# nice UI
echo_info('Choose from following scopes:')
for match_dict in scope_matcher:
echo_info(
"{number} | {scope_id:.8} | {scope}".format(**match_dict))
scope_match = None
while not scope_match:
scope_guess = click.prompt('Row number, part of Id or Scope')
scope_match = validate_scopes(scope_guess, scope_matcher)
echo_success(
"Scope selected: '{scope}' ({scope_id})".format(**scope_match))
scope_id = scope_match['scope_id']
scope_to_upload = get_project(url,
username,
password,
token,
scope_id=scope_id)
# service reupload
service_id = options.get('service_id') or settings.get('service_id')
if options.get('reupload') and not service_id:
echo_failure('Please provide a service id to reupload to.')
elif service_id and not options.get('reupload') and options.get(
'interactive'):
if click.confirm(
"Do you wish to *replace* the previously uploaded service: `{}`"
.format(service_id),
default=True):
service_id = service_id
else:
service_id = None
# store to settings
if options.get('store'):
settings.update(
dict(url=url, username=username, scope_id=str(scope_id)))
if service_id:
settings['service_id'] = str(service_id)
save_settings(settings,
settings_filename=options.get('settings_filename'))
# do upload
build_path = os.path.join(get_package_dir(package_name),
settings.get('build_dir'))
if not os.path.exists(build_path):
echo_failure('Cannot find build path, please do `kecpkg build` first')
sys.exit(400)
upload_package(scope_to_upload,
build_path,
kecpkg,
service_id=service_id,
settings=settings,
store_settings=options.get('store'),
settings_filename=options.get('settings_filename'))
def upload_package(scope,
build_path=None,
kecpkg_path=None,
service_id=None,
settings=None,
store_settings=True,
settings_filename=None):
"""
Upload the package from build_path to the right scope, create a new KE-chain SIM service.
:param scope: Scope object (pykechain)
:param build_path: path to the build directory in which the to-be uploaded script resides
:param kecpkg_path: path to the kecpkg file to upload (no need to provide build_path)
:param service_id: UUID of the service to upload to
:param settings: settings of the package
:param store_settings: store the settings after update (eg service_id after upload)
:param settings_filename: pathname of the file where the settings are stored
:return: None
"""
# if not (kecpkg_path and not build_path) or not (build_path and not kecpkg_path):
# echo_failure("You should provide a build path or a kecpkg path")
# sys.exit(404)
if kecpkg_path and os.path.exists(kecpkg_path):
kecpkg_path = kecpkg_path
else:
built_kecpkgs = os.listdir(build_path)
if not kecpkg_path and len(built_kecpkgs) > 1 and settings.get(
'version'):
built_kecpkgs = [
f for f in built_kecpkgs if settings.get('version') in f
]
if not kecpkg_path and len(built_kecpkgs) == 1:
kecpkg_path = os.path.join(build_path, built_kecpkgs[0])
else:
echo_info('Provide correct filename to upload')
echo_info('\n'.join(os.listdir(build_path)))
kecpkg_filename = click.prompt('Filename')
kecpkg_path = os.path.join(build_path, kecpkg_filename)
if kecpkg_path and os.path.exists(kecpkg_path):
# ready to upload
echo_info('Ready to upload `{}`'.format(os.path.basename(kecpkg_path)))
else:
echo_failure('Unable to locate kecpkg to upload')
sys.exit(404)
# get meta and prepare 2 stage submission
# 1. fill service information
# 2. do upload
if service_id:
service = scope.service(pk=service_id)
service.upload(kecpkg_path)
service.edit(name=settings.get('package_name'),
description=settings.get('description', ''),
script_version=settings.get('version', ''))
else:
# Create new service in KE-chain
service = scope.create_service(
name=settings.get('package_name'),
description=settings.get('description', ''),
version=settings.get('version', ''),
service_type='PYTHON SCRIPT',
environment_version=settings.get('python_version'),
pkg_path=kecpkg_path)
# Wrap up party!
echo_success("kecpkg `{}` successfully uploaded to KE-chain.".format(
os.path.basename(kecpkg_path)))
# noinspection PyProtectedMember
success_url = "{api_root}/#scopes/{scope_id}/scripts/{service_id}".format(
api_root=scope._client.api_root,
scope_id=scope.id,
service_id=service.id)
echo_success(
"To view the newly created service, go to: `{}`".format(success_url))
# update settings
if store_settings:
settings['service_id'] = str(service.id)
from datetime import datetime
settings['last_upload'] = str(datetime.now().isoformat())
save_settings(settings, settings_filename=settings_filename)