def _do_delete_key(gpg, options):
echo_info(
"Deleting private key with ID '{}' from the KECPKG keyring".format(
options.get('do_delete_key')))
# custom call to gpg using --delete-secret-and-public-key
result = gpg.result_map['delete'](gpg)
# noinspection PyProtectedMember
p = gpg._open_subprocess([
'--yes', '--delete-secret-and-public-key',
options.get('do_delete_key')
])
# noinspection PyProtectedMember
gpg._collect_output(p, result, stdin=p.stdin)
# result = gpg.delete_keys(fingerprints=options.get('do_delete_key'),
# secret=True,
# passphrase=options.get('sign_passphrase'))
# pprint(result.__dict__)
if result and result.stderr.find("failed") < 0:
echo_success("Succesfully deleted key")
_do_list(gpg=gpg)
sys.exit(0)
echo_failure("Could not delete key.")
sys.exit(1)
def get_gpg():
# type: () -> gnupg.GPG
"""Return the GPG objects instantiated with custom KECPKG keyring in custom KECPKG GNUPG home."""
global __gpg
if not __gpg:
if six.PY2:
echo_failure(
'Package signing capability is not available in python 2.7. Please use python 3 or greater.'
)
sys.exit(1)
import gnupg
logging.basicConfig(level=LOGLEVEL)
logging.getLogger('gnupg')
gpg_bin = 'gpg'
if ON_LINUX:
gpg_bin = subprocess.getoutput('which gpg')
if ON_WINDOWS:
bin_path_guesses = [
"C:\\Program Files (x86)\\GnuPG\\bin\\gpg.exe",
"C:\\Program Files\\GnuPG\\gpg.exe",
"C:\\Program Files (x86)\\GnuPG\\gpg.exe",
"C:\\Program Files\\GnuPG\\bin\\gpg.exe"
]
gpg_bins = [p for p in bin_path_guesses if os.path.exists(p)]
if gpg_bins is not None:
gpg_bin = gpg_bins[0]
else:
gpg_bin = bin_path_guesses[0]
elif ON_MACOS:
gpg_bin = '/usr/local/bin/gpg'
if not os.path.exists(gpg_bin):
echo_failure(
"Unable to detect installed GnuPG executable. Ensure you have it installed. "
"We checked: '{}'".format(gpg_bin))
echo_failure(
"- For Linux please install GnuPG using your package manager. In Ubuntu/Debian this can be "
"achieved with `sudo apt install gnupg`.")
echo_failure(
"- For Mac OSX please install GnuPG using `brew install gpg`.")
echo_failure(
"- For Windows please install GnuPG using the downloads via: https://gnupg.org/download/"
)
sys.exit(1)
if not os.path.exists(GNUPG_KECPKG_HOME):
# create the GNUPG_KECPKG_HOME when not exist, otherwise the GPG will fail
ensure_dir_exists(GNUPG_KECPKG_HOME)
__gpg = gnupg.GPG(gpgbinary=gpg_bin, gnupghome=GNUPG_KECPKG_HOME)
return __gpg
def _do_clear(options):
echo_info("Clearing all keys from the KECPKG keyring")
if not options.get('do_yes'):
options['do_yes'] = click.confirm(
"Are you sure you want to clear the KECPKG keyring?",
default=False)
if options.get('do_yes'):
remove_path(GNUPG_KECPKG_HOME)
echo_success("Completed")
sys.exit(0)
else:
echo_failure("Not removing the KECPKG keyring")
sys.exit(1)
def verify_signature(package_dir, artifacts_filename, artifacts_sig_filename):
"""
Check signature of the package.
:param package_dir: directory fullpath of the package
:param artifacts_filename: path of the artifacts file
:param artifacts_sig_filename: path of the artifacts signature file
:return: None
"""
gpg = get_gpg()
artifacts_fp = os.path.join(package_dir, artifacts_filename)
artifacts_sig_fp = os.path.join(package_dir, artifacts_sig_filename)
if not os.path.exists(artifacts_fp):
echo_failure(
"Artifacts file does not exist: '{}'".format(artifacts_filename))
sys.exit(1)
if not os.path.exists(artifacts_sig_fp):
echo_failure(
"Artifacts signature file does not exist: '{}'. Is the package signed?"
.format(artifacts_filename))
sys.exit(1)
with open(artifacts_sig_fp, 'rb') as sig_fd:
results = gpg.verify_file(sig_fd, data_filename=artifacts_fp)
if results.valid:
echo_info("Verified the signature and the signature is valid")
echo_info("Signed with: '{}'".format(results.username))
elif not results.valid:
echo_failure("Signature of the package is invalid")
echo_failure(pprint(results.__dict__))
sys.exit(1)
def pip_install_venv(package_dir, settings, verbose=False):
"""
Install requirements into the virtual environment.
:param package_dir: the full path to the package directory
:param settings: the settings dict (incluing the venv_dir name)
:param verbose: (optional) be more verbose if set to True, defaults to False
"""
venv_dir = os.path.join(package_dir, settings.get('venv_dir'))
if not os.path.exists(venv_dir):
echo_failure(
'virtual environment directory `{}` does not exists, nothing to install'
.format(venv_dir))
sys.exit(1)
if not os.path.exists(
os.path.join(package_dir, settings.get('requirements_filename'))):
echo_failure(
'could not find requirements.txt to install, check if `{}` exists or update settings'
.format(settings.get('requirements_filename')))
sys.exit(1)
install_command = [
sys.executable, '-m', 'pip', 'install', '-r',
os.path.join(package_dir, settings.get('requirements_filename'))
]
if not verbose: # no cov
install_command.append('-qqq')
with venv(venv_dir):
echo_info(
'Installing requirements from `{}` into the virtual environment `{}`'
.format(settings.get('requirements_filename'),
settings.get('venv_dir')))
result = None
if six.PY3:
result = subprocess.run(install_command,
shell=NEED_SUBPROCESS_SHELL)
return result.returncode
elif six.PY2:
result = subprocess.check_output(install_command,
shell=NEED_SUBPROCESS_SHELL)
return result and 0 or -1
if result:
echo_success(str(result))
return result.returncode
def sign_package(package_dir, settings, options=None, verbose=False):
"""
Sign the package with a GPG/PGP key.
:param package_dir: directory fullpath of the package
:param settings: settings object
:param options: commandline options dictionary passed down.
:param verbose: be verbose (or not)
:return: None
"""
gpg = get_gpg()
if options.get('sign_keyid') is None:
tabulate_keys(gpg, explain=True)
options['sign_keyid'] = click.prompt(
"Provide Key (Name, Comment, Email, Fingerprint) to sign package with",
default=settings.get('email'))
if options.get('sign_passphrase') is None:
options['sign_passphrase'] = click.prompt("Provide Passphrase",
hide_input=True)
echo_info('Signing package contents')
with open(
os.path.join(
package_dir,
settings.get('artifacts_filename', ARTIFACTS_FILENAME)),
'rb') as fd:
results = gpg.sign_file(fd,
keyid=options.get('sign_keyid'),
passphrase=options.get('sign_passphrase'),
detach=True,
output=settings.get('artifacts_sig_filename',
ARTIFACTS_SIG_FILENAME))
pprint(results.__dict__)
if results and results.status is not None:
echo_info("Signed package contents: {}".format(results.status))
else:
failure_text = results.stderr.split("\n")[-2]
echo_failure(
"Could not sign the package contents: '{}'".format(failure_text))
sys.exit(1)
if verbose:
echo_success('Successfully signed the package contents.')
verify_signature(package_dir, ARTIFACTS_FILENAME, ARTIFACTS_SIG_FILENAME)
verify_artifacts_hashes(package_dir, ARTIFACTS_FILENAME)
def verify_artifacts_hashes(package_dir, artifacts_filename):
"""
Check the hashes of the artifacts in the package.
:param package_dir: directory fullpath of the package
:param artifacts_filename: filename of the artifacts file
:return:
"""
artifacts_fp = os.path.join(package_dir, artifacts_filename)
if not os.path.exists(artifacts_fp):
echo_failure(
"Artifacts file does not exist: '{}'".format(artifacts_filename))
sys.exit(1)
with open(artifacts_fp, 'r') as fd:
artifacts = fd.readlines()
# process the file contents
# A line is "README.md,sha256=d831....ccf79a,336"
# ^filename ^algo ^hash ^size in bytes
fails = []
for af in artifacts:
# noinspection PyShadowingBuiltins,PyShadowingBuiltins
filename, hash, orig_size = af.split(',')
algorithm, orig_hash = hash.split('=')
fp = os.path.join(package_dir, filename)
if os.path.exists(fp):
found_hash = hash_of_file(fp, algorithm)
found_size = os.stat(fp).st_size
if found_hash != orig_hash.strip() or found_size != int(
orig_size.strip()):
fails.append(
"File '{}' is changed in the package.".format(filename))
fails.append(
"File '{}' original checksum: '{}', found: '{}'".format(
filename, orig_hash, found_hash))
fails.append("File '{}' original size: {}, found: {}".format(
filename, orig_size, found_size))
else:
fails.append("File '{}' does not exist".format(filename))
if fails:
echo_failure(
'The package has been changed after building the package.')
for fail in fails:
print(fail)
sys.exit(1)
else:
echo_info("Package contents succesfully verified.")
def load_settings(lazy=False, package_dir=None, settings_filename=None):
"""
Load settings from disk.
:param lazy: (optional) does lazy loading (default to False)
:param package_dir: (optional) loads the settings from a package dir
:param settings_filename: (optional) pathname of the file where the settings are stored
:return: settings dictionary
"""
settings_filepath = get_settings_filepath(package_dir, settings_filename)
if lazy and not os.path.exists(settings_filepath):
return {}
elif not os.path.exists(settings_filepath):
echo_failure('Could not find a settingsfile in path: {}'.format(
settings_filepath))
sys.exit(404)
else:
with open(settings_filepath, 'r') as f:
return json.loads(f.read(), object_pairs_hook=OrderedDict)
def prune(package, **options):
"""Remove a project's build artifacts."""
package_name = package or get_package_name() or click.prompt('Provide package name')
package_dir = get_package_dir(package_name)
settings = load_settings(package_name)
# ensure build directory is there
build_dir = settings.get('build_dir', 'dist')
build_path = os.path.join(package_dir, build_dir)
if os.path.exists(build_path):
if options.get('force') or click.confirm(
"Do you want to prune build artifacts for package '{}'?".format(package_name)):
remove_path(build_path)
if os.path.exists(build_path):
echo_failure('Something went wrong pruning pacakage `{}`'.format(package_name))
else:
echo_warning('Package `{}` will not be pruned'.format(package_name))
else:
echo_failure('Package `{}` does not exist'.format(package_name))
def _do_export_key(gpg, options):
"""Export public key."""
echo_info("Exporting public key")
if options.get('keyid') is None:
_do_list(gpg=gpg)
options['keyid'] = click.prompt(
"Provide KeyId (name, comment, email, fingerprint) of the key to export"
)
result = gpg.export_keys(keyids=[options.get('keyid')],
secret=False,
armor=True)
if result is not None:
with open(options.get('do_export_key'), 'w') as fd:
fd.write(result)
echo_success("Sucessfully written public key to '{}'".format(
options.get('do_export_key')))
sys.exit(0)
echo_failure("Could not export key")
sys.exit(1)
def purge(package, **options):
"""
Purge and clean a package directory structure.
:param package: Name of the kecpkg package
:param options:
:return:
"""
package_name = package or click.prompt('Provide package name')
package_dir = get_package_dir(package_name)
if os.path.exists(package_dir):
if options.get('force') or click.confirm(
"Do you want to purge and completely remove '{}'?".format(package_name)):
remove_path(package_dir)
if not os.path.exists(package_dir):
echo_success('Package `{}` is purged and removed from disk'.format(package_name))
else:
echo_failure('Something went wrong pruning pacakage `{}`'.format(package_name))
else:
echo_warning('Package `{}` will not be purged'.format(package_name))
else:
echo_failure('Package `{}` does not exist'.format(package_name))
def _do_import(gpg, options):
echo_info("Importing secret key into KECPKG keyring from '{}'".format(
options.get('do_import')))
result = gpg.import_keys(
open(os.path.abspath(options.get('do_import')), 'rb').read())
# pprint(result.__dict__)
if result and result.sec_imported:
echo_success(
"Succesfully imported secret key into the KECPKG keystore")
_do_list(gpg=gpg)
sys.exit(0)
elif result and result.unchanged:
echo_failure(
"Did not import the secret key into the KECPKG keystore. The key was already "
"in place and was unchanged")
_do_list(gpg=gpg)
sys.exit(1)
echo_failure(
"Did not import a secret key into the KECPKG keystore. Is something wrong "
"with the file: '{}'? Are you sure it is a ASCII file containing a "
"private key block?".format(options.get('do_import')))
sys.exit(1)
def new(package=None, **options):
"""
Create a new package directory structure.
<pkg dir>
+-- venv
| +-- ... <the virtualenvironment>
+-- README.md
+-- requirements.txt
+-- script.py
+-- package-info.json
+-- .gitignore
+-- .kecpkg-settings.json
"""
settings = load_settings(lazy=True, settings_filename=options.get('settings_filename'))
if not settings:
settings = copy_default_settings()
package_root_dir = os.getcwd()
# set the package name, clean an normalise using snake_case
package_name = package or click.prompt("Package name")
package_name = normalise_name(package_name)
# save to settings
settings['package_name'] = package_name
package_dir = os.path.join(package_root_dir, package_name)
if os.path.exists(package_dir):
echo_failure("Directory '{}' already exists.".format(package_dir))
sys.exit(1)
if not package:
settings['version'] = click.prompt('Version', default=settings.get('version', '0.0.1'))
settings['description'] = click.prompt('Description', default='')
settings['name'] = click.prompt('Author', default=settings.get('name', os.environ.get('USER', '')))
settings['email'] = click.prompt('Author\'s email', default=settings.get('email', ''))
settings['python_version'] = click.prompt('Python version (choose from: {})'.format(settings.get('pyversions')),
default='3.5')
settings['exclude_paths'] = click.prompt("Exclude additional paths from kecpkg (eg. 'data, input')",
default=settings.get('exclude_paths', ''),
value_proc=process_additional_exclude_paths)
if options.get('script'):
script_base = normalise_name(options.get('script').replace('.py', ''))
echo_info('Setting the script to `{}`'.format(script_base))
settings['entrypoint_script'] = script_base
if options.get('venv'):
settings['venv_dir'] = normalise_name(options.get('venv'))
echo_info("Creating package structure")
create_package(package_dir, settings=settings)
if not options.get('no_venv'):
echo_info("Creating virtual environment")
create_venv(package_dir, settings, pypath=None, use_global=options.get('global_packages'),
verbose=options.get('verbose'))
pip_install_venv(package_dir, settings, verbose=options.get('verbose'))
else:
settings['venv_dir'] = None
# save the settings (in the package_dir)
save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename'))
echo_success('Package `{package_name}` created in `{package_dir}`'.format(package_name=package_name,
package_dir=package_dir))
def _do_create_key(gpg, options):
echo_info(
"Will create a secret key and store it into the KECPKG keyring.")
package_dir = get_package_dir(package_name=package, fail=False)
settings = DEFAULT_SETTINGS
if package_dir is not None:
package_name = os.path.basename(package_dir)
echo_info('Package `{}` has been selected'.format(package_name))
settings = load_settings(
package_dir=package_dir,
settings_filename=options.get('settings_filename'))
key_info = {
'name_real':
click.prompt("Name", default=settings.get('name')),
'name_comment':
click.prompt("Comment", default="KECPKG SIGNING KEY"),
'name_email':
click.prompt("Email", default=settings.get('email')),
'expire_date':
click.prompt("Expiration in months",
default=12,
value_proc=lambda i: "{}m".format(i)),
'key_type':
'RSA',
'key_length':
4096,
'key_usage':
'',
'subkey_type':
'RSA',
'subkey_length':
4096,
'subkey_usage':
'encrypt,sign,auth',
'passphrase':
''
}
passphrase = click.prompt("Passphrase", hide_input=True)
passphrase_confirmed = click.prompt("Confirm passphrase",
hide_input=True)
if passphrase == passphrase_confirmed:
key_info['passphrase'] = passphrase
else:
raise ValueError("The passphrases did not match.")
echo_info(
"Creating the secret key '{name_real} ({name_comment}) <{name_email}>'"
.format(**key_info))
echo_info(
"Please move around mouse or generate other activity to introduce sufficient entropy. "
"This might take a minute...")
result = gpg.gen_key(gpg.gen_key_input(**key_info))
pprint(result.__dict__)
if result and result.stderr.find('KEY_CREATED'):
echo_success("The key is succesfully created")
_do_list(gpg=gpg)
sys.exit(0)
echo_failure("Could not generate the key due to an error: '{}'".format(
result.stderr))
sys.exit(1)
def upload(package=None,
url=None,
username=None,
password=None,
token=None,
scope=None,
scope_id=None,
kecpkg=None,
**options):
"""
Upload built kecpkg to KE-chain.
If no options are provided, the interactive mode is triggered.
"""
package_name = package or get_package_name() or click.prompt(
'Package name')
settings = load_settings(
package_dir=get_package_dir(package_name),
settings_filename=options.get('settings_filename'))
if not url or not ((username and password) or token):
url = click.prompt('Url (incl http(s)://)',
default=settings.get('url') or url)
username = click.prompt('Username',
default=settings.get('username') or username)
password = click.prompt('Password', hide_input=True)
# set the interactive world to True for continuation sake
options['interactive'] = True
elif not options.get('interactive'):
url = url or settings.get('url')
username = username or settings.get('username')
token = token or settings.get('token')
scope_id = scope_id or settings.get('scope_id')
client = Client(url)
client.login(username=username, password=password, token=token)
# scope finder
if not scope_id and settings.get('scope_id') and \
click.confirm("Do you wish to use the stored `scope_id` in settings: `{}`".format(
settings.get('scope_id')), default=True):
scope_id = settings.get('scope_id')
if not scope_id:
scopes = client.scopes()
scope_matcher = [
dict(number=i, scope_id=scope.id, scope=scope.name)
for i, scope in zip(range(1, len(scopes)), scopes)
]
# nice UI
echo_info('Choose from following scopes:')
for match_dict in scope_matcher:
echo_info(
"{number} | {scope_id:.8} | {scope}".format(**match_dict))
scope_match = None
while not scope_match:
scope_guess = click.prompt('Row number, part of Id or Scope')
scope_match = validate_scopes(scope_guess, scope_matcher)
echo_success(
"Scope selected: '{scope}' ({scope_id})".format(**scope_match))
scope_id = scope_match['scope_id']
scope_to_upload = get_project(url,
username,
password,
token,
scope_id=scope_id)
# service reupload
service_id = options.get('service_id') or settings.get('service_id')
if options.get('reupload') and not service_id:
echo_failure('Please provide a service id to reupload to.')
elif service_id and not options.get('reupload') and options.get(
'interactive'):
if click.confirm(
"Do you wish to *replace* the previously uploaded service: `{}`"
.format(service_id),
default=True):
service_id = service_id
else:
service_id = None
# store to settings
if options.get('store'):
settings.update(
dict(url=url, username=username, scope_id=str(scope_id)))
if service_id:
settings['service_id'] = str(service_id)
save_settings(settings,
settings_filename=options.get('settings_filename'))
# do upload
build_path = os.path.join(get_package_dir(package_name),
settings.get('build_dir'))
if not os.path.exists(build_path):
echo_failure('Cannot find build path, please do `kecpkg build` first')
sys.exit(400)
upload_package(scope_to_upload,
build_path,
kecpkg,
service_id=service_id,
settings=settings,
store_settings=options.get('store'),
settings_filename=options.get('settings_filename'))
def upload_package(scope,
build_path=None,
kecpkg_path=None,
service_id=None,
settings=None,
store_settings=True,
settings_filename=None):
"""
Upload the package from build_path to the right scope, create a new KE-chain SIM service.
:param scope: Scope object (pykechain)
:param build_path: path to the build directory in which the to-be uploaded script resides
:param kecpkg_path: path to the kecpkg file to upload (no need to provide build_path)
:param service_id: UUID of the service to upload to
:param settings: settings of the package
:param store_settings: store the settings after update (eg service_id after upload)
:param settings_filename: pathname of the file where the settings are stored
:return: None
"""
# if not (kecpkg_path and not build_path) or not (build_path and not kecpkg_path):
# echo_failure("You should provide a build path or a kecpkg path")
# sys.exit(404)
if kecpkg_path and os.path.exists(kecpkg_path):
kecpkg_path = kecpkg_path
else:
built_kecpkgs = os.listdir(build_path)
if not kecpkg_path and len(built_kecpkgs) > 1 and settings.get(
'version'):
built_kecpkgs = [
f for f in built_kecpkgs if settings.get('version') in f
]
if not kecpkg_path and len(built_kecpkgs) == 1:
kecpkg_path = os.path.join(build_path, built_kecpkgs[0])
else:
echo_info('Provide correct filename to upload')
echo_info('\n'.join(os.listdir(build_path)))
kecpkg_filename = click.prompt('Filename')
kecpkg_path = os.path.join(build_path, kecpkg_filename)
if kecpkg_path and os.path.exists(kecpkg_path):
# ready to upload
echo_info('Ready to upload `{}`'.format(os.path.basename(kecpkg_path)))
else:
echo_failure('Unable to locate kecpkg to upload')
sys.exit(404)
# get meta and prepare 2 stage submission
# 1. fill service information
# 2. do upload
if service_id:
service = scope.service(pk=service_id)
service.upload(kecpkg_path)
service.edit(name=settings.get('package_name'),
description=settings.get('description', ''),
script_version=settings.get('version', ''))
else:
# Create new service in KE-chain
service = scope.create_service(
name=settings.get('package_name'),
description=settings.get('description', ''),
version=settings.get('version', ''),
service_type='PYTHON SCRIPT',
environment_version=settings.get('python_version'),
pkg_path=kecpkg_path)
# Wrap up party!
echo_success("kecpkg `{}` successfully uploaded to KE-chain.".format(
os.path.basename(kecpkg_path)))
# noinspection PyProtectedMember
success_url = "{api_root}/#scopes/{scope_id}/scripts/{service_id}".format(
api_root=scope._client.api_root,
scope_id=scope.id,
service_id=service.id)
echo_success(
"To view the newly created service, go to: `{}`".format(success_url))
# update settings
if store_settings:
settings['service_id'] = str(service.id)
from datetime import datetime
settings['last_upload'] = str(datetime.now().isoformat())
save_settings(settings, settings_filename=settings_filename)