def build(package=None, **options):
"""Build the package and create a kecpkg file."""
echo_info('Locating package ``'.format(package))
package_dir = get_package_dir(package_name=package)
package_name = os.path.basename(package_dir)
echo_info('Package `{}` has been selected'.format(package_name))
settings = load_settings(
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
# ensure build directory is there
build_dir = settings.get('build_dir', 'dist')
build_path = os.path.join(package_dir, build_dir)
if options.get('update_package_info'):
render_package_info(settings, package_dir=package_dir, backup=True)
if options.get('clean_first'):
remove_path(build_path)
ensure_dir_exists(build_path)
# do package building
build_package(package_dir,
build_path,
settings,
options=options,
verbose=options.get('verbose'))
echo_success('Complete')
def _do_delete_key(gpg, options):
echo_info(
"Deleting private key with ID '{}' from the KECPKG keyring".format(
options.get('do_delete_key')))
# custom call to gpg using --delete-secret-and-public-key
result = gpg.result_map['delete'](gpg)
# noinspection PyProtectedMember
p = gpg._open_subprocess([
'--yes', '--delete-secret-and-public-key',
options.get('do_delete_key')
])
# noinspection PyProtectedMember
gpg._collect_output(p, result, stdin=p.stdin)
# result = gpg.delete_keys(fingerprints=options.get('do_delete_key'),
# secret=True,
# passphrase=options.get('sign_passphrase'))
# pprint(result.__dict__)
if result and result.stderr.find("failed") < 0:
echo_success("Succesfully deleted key")
_do_list(gpg=gpg)
sys.exit(0)
echo_failure("Could not delete key.")
sys.exit(1)
def _do_clear(options):
echo_info("Clearing all keys from the KECPKG keyring")
if not options.get('do_yes'):
options['do_yes'] = click.confirm(
"Are you sure you want to clear the KECPKG keyring?",
default=False)
if options.get('do_yes'):
remove_path(GNUPG_KECPKG_HOME)
echo_success("Completed")
sys.exit(0)
else:
echo_failure("Not removing the KECPKG keyring")
sys.exit(1)
def pip_install_venv(package_dir, settings, verbose=False):
"""
Install requirements into the virtual environment.
:param package_dir: the full path to the package directory
:param settings: the settings dict (incluing the venv_dir name)
:param verbose: (optional) be more verbose if set to True, defaults to False
"""
venv_dir = os.path.join(package_dir, settings.get('venv_dir'))
if not os.path.exists(venv_dir):
echo_failure(
'virtual environment directory `{}` does not exists, nothing to install'
.format(venv_dir))
sys.exit(1)
if not os.path.exists(
os.path.join(package_dir, settings.get('requirements_filename'))):
echo_failure(
'could not find requirements.txt to install, check if `{}` exists or update settings'
.format(settings.get('requirements_filename')))
sys.exit(1)
install_command = [
sys.executable, '-m', 'pip', 'install', '-r',
os.path.join(package_dir, settings.get('requirements_filename'))
]
if not verbose: # no cov
install_command.append('-qqq')
with venv(venv_dir):
echo_info(
'Installing requirements from `{}` into the virtual environment `{}`'
.format(settings.get('requirements_filename'),
settings.get('venv_dir')))
result = None
if six.PY3:
result = subprocess.run(install_command,
shell=NEED_SUBPROCESS_SHELL)
return result.returncode
elif six.PY2:
result = subprocess.check_output(install_command,
shell=NEED_SUBPROCESS_SHELL)
return result and 0 or -1
if result:
echo_success(str(result))
return result.returncode
def sign_package(package_dir, settings, options=None, verbose=False):
"""
Sign the package with a GPG/PGP key.
:param package_dir: directory fullpath of the package
:param settings: settings object
:param options: commandline options dictionary passed down.
:param verbose: be verbose (or not)
:return: None
"""
gpg = get_gpg()
if options.get('sign_keyid') is None:
tabulate_keys(gpg, explain=True)
options['sign_keyid'] = click.prompt(
"Provide Key (Name, Comment, Email, Fingerprint) to sign package with",
default=settings.get('email'))
if options.get('sign_passphrase') is None:
options['sign_passphrase'] = click.prompt("Provide Passphrase",
hide_input=True)
echo_info('Signing package contents')
with open(
os.path.join(
package_dir,
settings.get('artifacts_filename', ARTIFACTS_FILENAME)),
'rb') as fd:
results = gpg.sign_file(fd,
keyid=options.get('sign_keyid'),
passphrase=options.get('sign_passphrase'),
detach=True,
output=settings.get('artifacts_sig_filename',
ARTIFACTS_SIG_FILENAME))
pprint(results.__dict__)
if results and results.status is not None:
echo_info("Signed package contents: {}".format(results.status))
else:
failure_text = results.stderr.split("\n")[-2]
echo_failure(
"Could not sign the package contents: '{}'".format(failure_text))
sys.exit(1)
if verbose:
echo_success('Successfully signed the package contents.')
verify_signature(package_dir, ARTIFACTS_FILENAME, ARTIFACTS_SIG_FILENAME)
verify_artifacts_hashes(package_dir, ARTIFACTS_FILENAME)
def _do_export_key(gpg, options):
"""Export public key."""
echo_info("Exporting public key")
if options.get('keyid') is None:
_do_list(gpg=gpg)
options['keyid'] = click.prompt(
"Provide KeyId (name, comment, email, fingerprint) of the key to export"
)
result = gpg.export_keys(keyids=[options.get('keyid')],
secret=False,
armor=True)
if result is not None:
with open(options.get('do_export_key'), 'w') as fd:
fd.write(result)
echo_success("Sucessfully written public key to '{}'".format(
options.get('do_export_key')))
sys.exit(0)
echo_failure("Could not export key")
sys.exit(1)
def purge(package, **options):
"""
Purge and clean a package directory structure.
:param package: Name of the kecpkg package
:param options:
:return:
"""
package_name = package or click.prompt('Provide package name')
package_dir = get_package_dir(package_name)
if os.path.exists(package_dir):
if options.get('force') or click.confirm(
"Do you want to purge and completely remove '{}'?".format(package_name)):
remove_path(package_dir)
if not os.path.exists(package_dir):
echo_success('Package `{}` is purged and removed from disk'.format(package_name))
else:
echo_failure('Something went wrong pruning pacakage `{}`'.format(package_name))
else:
echo_warning('Package `{}` will not be purged'.format(package_name))
else:
echo_failure('Package `{}` does not exist'.format(package_name))
def _do_import(gpg, options):
echo_info("Importing secret key into KECPKG keyring from '{}'".format(
options.get('do_import')))
result = gpg.import_keys(
open(os.path.abspath(options.get('do_import')), 'rb').read())
# pprint(result.__dict__)
if result and result.sec_imported:
echo_success(
"Succesfully imported secret key into the KECPKG keystore")
_do_list(gpg=gpg)
sys.exit(0)
elif result and result.unchanged:
echo_failure(
"Did not import the secret key into the KECPKG keystore. The key was already "
"in place and was unchanged")
_do_list(gpg=gpg)
sys.exit(1)
echo_failure(
"Did not import a secret key into the KECPKG keystore. Is something wrong "
"with the file: '{}'? Are you sure it is a ASCII file containing a "
"private key block?".format(options.get('do_import')))
sys.exit(1)
def new(package=None, **options):
"""
Create a new package directory structure.
<pkg dir>
+-- venv
| +-- ... <the virtualenvironment>
+-- README.md
+-- requirements.txt
+-- script.py
+-- package-info.json
+-- .gitignore
+-- .kecpkg-settings.json
"""
settings = load_settings(lazy=True, settings_filename=options.get('settings_filename'))
if not settings:
settings = copy_default_settings()
package_root_dir = os.getcwd()
# set the package name, clean an normalise using snake_case
package_name = package or click.prompt("Package name")
package_name = normalise_name(package_name)
# save to settings
settings['package_name'] = package_name
package_dir = os.path.join(package_root_dir, package_name)
if os.path.exists(package_dir):
echo_failure("Directory '{}' already exists.".format(package_dir))
sys.exit(1)
if not package:
settings['version'] = click.prompt('Version', default=settings.get('version', '0.0.1'))
settings['description'] = click.prompt('Description', default='')
settings['name'] = click.prompt('Author', default=settings.get('name', os.environ.get('USER', '')))
settings['email'] = click.prompt('Author\'s email', default=settings.get('email', ''))
settings['python_version'] = click.prompt('Python version (choose from: {})'.format(settings.get('pyversions')),
default='3.5')
settings['exclude_paths'] = click.prompt("Exclude additional paths from kecpkg (eg. 'data, input')",
default=settings.get('exclude_paths', ''),
value_proc=process_additional_exclude_paths)
if options.get('script'):
script_base = normalise_name(options.get('script').replace('.py', ''))
echo_info('Setting the script to `{}`'.format(script_base))
settings['entrypoint_script'] = script_base
if options.get('venv'):
settings['venv_dir'] = normalise_name(options.get('venv'))
echo_info("Creating package structure")
create_package(package_dir, settings=settings)
if not options.get('no_venv'):
echo_info("Creating virtual environment")
create_venv(package_dir, settings, pypath=None, use_global=options.get('global_packages'),
verbose=options.get('verbose'))
pip_install_venv(package_dir, settings, verbose=options.get('verbose'))
else:
settings['venv_dir'] = None
# save the settings (in the package_dir)
save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename'))
echo_success('Package `{package_name}` created in `{package_dir}`'.format(package_name=package_name,
package_dir=package_dir))
def _do_create_key(gpg, options):
echo_info(
"Will create a secret key and store it into the KECPKG keyring.")
package_dir = get_package_dir(package_name=package, fail=False)
settings = DEFAULT_SETTINGS
if package_dir is not None:
package_name = os.path.basename(package_dir)
echo_info('Package `{}` has been selected'.format(package_name))
settings = load_settings(
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
key_info = {
'name_real':
click.prompt("Name", default=settings.get('name')),
'name_comment':
click.prompt("Comment", default="KECPKG SIGNING KEY"),
'name_email':
click.prompt("Email", default=settings.get('email')),
'expire_date':
click.prompt("Expiration in months",
default=12,
value_proc=lambda i: "{}m".format(i)),
'key_type':
'RSA',
'key_length':
4096,
'key_usage':
'',
'subkey_type':
'RSA',
'subkey_length':
4096,
'subkey_usage':
'encrypt,sign,auth',
'passphrase':
''
}
passphrase = click.prompt("Passphrase", hide_input=True)
passphrase_confirmed = click.prompt("Confirm passphrase",
hide_input=True)
if passphrase == passphrase_confirmed:
key_info['passphrase'] = passphrase
else:
raise ValueError("The passphrases did not match.")
echo_info(
"Creating the secret key '{name_real} ({name_comment}) <{name_email}>'"
.format(**key_info))
echo_info(
"Please move around mouse or generate other activity to introduce sufficient entropy. "
"This might take a minute...")
result = gpg.gen_key(gpg.gen_key_input(**key_info))
pprint(result.__dict__)
if result and result.stderr.find('KEY_CREATED'):
echo_success("The key is succesfully created")
_do_list(gpg=gpg)
sys.exit(0)
echo_failure("Could not generate the key due to an error: '{}'".format(
result.stderr))
sys.exit(1)
def config(package, **options):
"""Manage the configuration (or settings) of the package.
The various settings in the .kecpkg-settings.json file are:
package_name: name of the package
version: version number of the package
description: longer description of the package
name: name of the author of the package
email: email of the author of the package
python_version: python version on which the kecpkg will run, corresponds
with an executable KE-crunch environment
entrypoint_script: the name of the script that will be executed first
entrypoint_func: function name inside the script that will be executed.
Ensure that it takes *args, **kwargs.
venv_dir: python virtual environment directory in the development environment
requirements_filename: name of the requirements file with list of package that
will be installed before running
build_dir: directory where the built kecpkg will be stored
exclude_paths: list of paths that will be excluded from the package, next to
the build in excludes
url: url where the package will be uploaded
token: token of the user under which the package is uploaded
scope_id: identification of the scope under which the package is uploaded
service_id: identification under which the package is re-uploaded
(or recently uploaded)
last_upload: date and time of the last upload
"""
echo_info('Locating package ``'.format(package))
package_dir = get_package_dir(package_name=package)
package_name = os.path.basename(package_dir)
echo_info('Package `{}` has been selected'.format(package_name))
if options.get('init'):
if os.path.exists(os.path.join(package_dir, options.get('settings_filename'))) and \
click.confirm('Are you sure you want to overwrite the current settingsfile '
'(old settings will be a backup)?'):
copy_path(
os.path.join(package_dir, options.get('settings_filename')),
os.path.join(
package_dir,
"{}-backup".format(options.get('settings_filename'))))
echo_info('Creating new settingsfile')
settings = copy_default_settings()
settings['package_name'] = package_name
save_settings(settings,
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
settings = load_settings(package_dir=package_dir)
if options.get('interactive'):
settings['version'] = click.prompt('Version',
default=settings.get(
'version', '0.0.1'))
settings['description'] = click.prompt('Description',
default=settings.get(
'description', ''))
settings['name'] = click.prompt('Author',
default=settings.get(
'name', os.environ.get('USER',
'')))
settings['email'] = click.prompt('Author\'s email',
default=settings.get('email', ''))
settings['python_version'] = click.prompt(
'Python version (choose from: {})'.format(
settings.get('pyversions')),
default='3.5')
settings['exclude_paths'] = click.prompt(
"Exclude additional paths from kecpkg (eg. 'data, input')",
default=settings.get('exclude_paths', ''),
value_proc=process_additional_exclude_paths)
save_settings(settings,
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
if options.get('set_key'):
k, v = options.get('set_key')
if options.get('verbose'):
echo_info("Set the key '{}' to value '{}'".format(k, v))
settings[k] = v
save_settings(settings,
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
if options.get('get_key'):
echo_info(
tabulate([
(options.get('get_key'), settings.get(options.get('get_key')))
],
headers=("key", "value")))
return
if options.get('verbose'):
echo_info(tabulate(settings.items(), headers=("key", "value")))
if not options.get('interactive'):
echo_success('Settings file identified and correct')
def upload(package=None,
url=None,
username=None,
password=None,
token=None,
scope=None,
scope_id=None,
kecpkg=None,
**options):
"""
Upload built kecpkg to KE-chain.
If no options are provided, the interactive mode is triggered.
"""
package_name = package or get_package_name() or click.prompt(
'Package name')
settings = load_settings(
package_dir=get_package_dir(package_name),
settings_filename=options.get('settings_filename'))
if not url or not ((username and password) or token):
url = click.prompt('Url (incl http(s)://)',
default=settings.get('url') or url)
username = click.prompt('Username',
default=settings.get('username') or username)
password = click.prompt('Password', hide_input=True)
# set the interactive world to True for continuation sake
options['interactive'] = True
elif not options.get('interactive'):
url = url or settings.get('url')
username = username or settings.get('username')
token = token or settings.get('token')
scope_id = scope_id or settings.get('scope_id')
client = Client(url)
client.login(username=username, password=password, token=token)
# scope finder
if not scope_id and settings.get('scope_id') and \
click.confirm("Do you wish to use the stored `scope_id` in settings: `{}`".format(
settings.get('scope_id')), default=True):
scope_id = settings.get('scope_id')
if not scope_id:
scopes = client.scopes()
scope_matcher = [
dict(number=i, scope_id=scope.id, scope=scope.name)
for i, scope in zip(range(1, len(scopes)), scopes)
]
# nice UI
echo_info('Choose from following scopes:')
for match_dict in scope_matcher:
echo_info(
"{number} | {scope_id:.8} | {scope}".format(**match_dict))
scope_match = None
while not scope_match:
scope_guess = click.prompt('Row number, part of Id or Scope')
scope_match = validate_scopes(scope_guess, scope_matcher)
echo_success(
"Scope selected: '{scope}' ({scope_id})".format(**scope_match))
scope_id = scope_match['scope_id']
scope_to_upload = get_project(url,
username,
password,
token,
scope_id=scope_id)
# service reupload
service_id = options.get('service_id') or settings.get('service_id')
if options.get('reupload') and not service_id:
echo_failure('Please provide a service id to reupload to.')
elif service_id and not options.get('reupload') and options.get(
'interactive'):
if click.confirm(
"Do you wish to *replace* the previously uploaded service: `{}`"
.format(service_id),
default=True):
service_id = service_id
else:
service_id = None
# store to settings
if options.get('store'):
settings.update(
dict(url=url, username=username, scope_id=str(scope_id)))
if service_id:
settings['service_id'] = str(service_id)
save_settings(settings,
settings_filename=options.get('settings_filename'))
# do upload
build_path = os.path.join(get_package_dir(package_name),
settings.get('build_dir'))
if not os.path.exists(build_path):
echo_failure('Cannot find build path, please do `kecpkg build` first')
sys.exit(400)
upload_package(scope_to_upload,
build_path,
kecpkg,
service_id=service_id,
settings=settings,
store_settings=options.get('store'),
settings_filename=options.get('settings_filename'))
def upload_package(scope,
build_path=None,
kecpkg_path=None,
service_id=None,
settings=None,
store_settings=True,
settings_filename=None):
"""
Upload the package from build_path to the right scope, create a new KE-chain SIM service.
:param scope: Scope object (pykechain)
:param build_path: path to the build directory in which the to-be uploaded script resides
:param kecpkg_path: path to the kecpkg file to upload (no need to provide build_path)
:param service_id: UUID of the service to upload to
:param settings: settings of the package
:param store_settings: store the settings after update (eg service_id after upload)
:param settings_filename: pathname of the file where the settings are stored
:return: None
"""
# if not (kecpkg_path and not build_path) or not (build_path and not kecpkg_path):
# echo_failure("You should provide a build path or a kecpkg path")
# sys.exit(404)
if kecpkg_path and os.path.exists(kecpkg_path):
kecpkg_path = kecpkg_path
else:
built_kecpkgs = os.listdir(build_path)
if not kecpkg_path and len(built_kecpkgs) > 1 and settings.get(
'version'):
built_kecpkgs = [
f for f in built_kecpkgs if settings.get('version') in f
]
if not kecpkg_path and len(built_kecpkgs) == 1:
kecpkg_path = os.path.join(build_path, built_kecpkgs[0])
else:
echo_info('Provide correct filename to upload')
echo_info('\n'.join(os.listdir(build_path)))
kecpkg_filename = click.prompt('Filename')
kecpkg_path = os.path.join(build_path, kecpkg_filename)
if kecpkg_path and os.path.exists(kecpkg_path):
# ready to upload
echo_info('Ready to upload `{}`'.format(os.path.basename(kecpkg_path)))
else:
echo_failure('Unable to locate kecpkg to upload')
sys.exit(404)
# get meta and prepare 2 stage submission
# 1. fill service information
# 2. do upload
if service_id:
service = scope.service(pk=service_id)
service.upload(kecpkg_path)
service.edit(name=settings.get('package_name'),
description=settings.get('description', ''),
script_version=settings.get('version', ''))
else:
# Create new service in KE-chain
service = scope.create_service(
name=settings.get('package_name'),
description=settings.get('description', ''),
version=settings.get('version', ''),
service_type='PYTHON SCRIPT',
environment_version=settings.get('python_version'),
pkg_path=kecpkg_path)
# Wrap up party!
echo_success("kecpkg `{}` successfully uploaded to KE-chain.".format(
os.path.basename(kecpkg_path)))
# noinspection PyProtectedMember
success_url = "{api_root}/#scopes/{scope_id}/scripts/{service_id}".format(
api_root=scope._client.api_root,
scope_id=scope.id,
service_id=service.id)
echo_success(
"To view the newly created service, go to: `{}`".format(success_url))
# update settings
if store_settings:
settings['service_id'] = str(service.id)
from datetime import datetime
settings['last_upload'] = str(datetime.now().isoformat())
save_settings(settings, settings_filename=settings_filename)
