def _do_delete_key(gpg, options): echo_info( "Deleting private key with ID '{}' from the KECPKG keyring".format( options.get('do_delete_key'))) # custom call to gpg using --delete-secret-and-public-key result = gpg.result_map['delete'](gpg) # noinspection PyProtectedMember p = gpg._open_subprocess([ '--yes', '--delete-secret-and-public-key', options.get('do_delete_key') ]) # noinspection PyProtectedMember gpg._collect_output(p, result, stdin=p.stdin) # result = gpg.delete_keys(fingerprints=options.get('do_delete_key'), # secret=True, # passphrase=options.get('sign_passphrase')) # pprint(result.__dict__) if result and result.stderr.find("failed") < 0: echo_success("Succesfully deleted key") _do_list(gpg=gpg) sys.exit(0) echo_failure("Could not delete key.") sys.exit(1)
def get_gpg(): # type: () -> gnupg.GPG """Return the GPG objects instantiated with custom KECPKG keyring in custom KECPKG GNUPG home.""" global __gpg if not __gpg: if six.PY2: echo_failure( 'Package signing capability is not available in python 2.7. Please use python 3 or greater.' ) sys.exit(1) import gnupg logging.basicConfig(level=LOGLEVEL) logging.getLogger('gnupg') gpg_bin = 'gpg' if ON_LINUX: gpg_bin = subprocess.getoutput('which gpg') if ON_WINDOWS: bin_path_guesses = [ "C:\\Program Files (x86)\\GnuPG\\bin\\gpg.exe", "C:\\Program Files\\GnuPG\\gpg.exe", "C:\\Program Files (x86)\\GnuPG\\gpg.exe", "C:\\Program Files\\GnuPG\\bin\\gpg.exe" ] gpg_bins = [p for p in bin_path_guesses if os.path.exists(p)] if gpg_bins is not None: gpg_bin = gpg_bins[0] else: gpg_bin = bin_path_guesses[0] elif ON_MACOS: gpg_bin = '/usr/local/bin/gpg' if not os.path.exists(gpg_bin): echo_failure( "Unable to detect installed GnuPG executable. Ensure you have it installed. " "We checked: '{}'".format(gpg_bin)) echo_failure( "- For Linux please install GnuPG using your package manager. In Ubuntu/Debian this can be " "achieved with `sudo apt install gnupg`.") echo_failure( "- For Mac OSX please install GnuPG using `brew install gpg`.") echo_failure( "- For Windows please install GnuPG using the downloads via: https://gnupg.org/download/" ) sys.exit(1) if not os.path.exists(GNUPG_KECPKG_HOME): # create the GNUPG_KECPKG_HOME when not exist, otherwise the GPG will fail ensure_dir_exists(GNUPG_KECPKG_HOME) __gpg = gnupg.GPG(gpgbinary=gpg_bin, gnupghome=GNUPG_KECPKG_HOME) return __gpg
def _do_clear(options): echo_info("Clearing all keys from the KECPKG keyring") if not options.get('do_yes'): options['do_yes'] = click.confirm( "Are you sure you want to clear the KECPKG keyring?", default=False) if options.get('do_yes'): remove_path(GNUPG_KECPKG_HOME) echo_success("Completed") sys.exit(0) else: echo_failure("Not removing the KECPKG keyring") sys.exit(1)
def verify_signature(package_dir, artifacts_filename, artifacts_sig_filename): """ Check signature of the package. :param package_dir: directory fullpath of the package :param artifacts_filename: path of the artifacts file :param artifacts_sig_filename: path of the artifacts signature file :return: None """ gpg = get_gpg() artifacts_fp = os.path.join(package_dir, artifacts_filename) artifacts_sig_fp = os.path.join(package_dir, artifacts_sig_filename) if not os.path.exists(artifacts_fp): echo_failure( "Artifacts file does not exist: '{}'".format(artifacts_filename)) sys.exit(1) if not os.path.exists(artifacts_sig_fp): echo_failure( "Artifacts signature file does not exist: '{}'. Is the package signed?" .format(artifacts_filename)) sys.exit(1) with open(artifacts_sig_fp, 'rb') as sig_fd: results = gpg.verify_file(sig_fd, data_filename=artifacts_fp) if results.valid: echo_info("Verified the signature and the signature is valid") echo_info("Signed with: '{}'".format(results.username)) elif not results.valid: echo_failure("Signature of the package is invalid") echo_failure(pprint(results.__dict__)) sys.exit(1)
def pip_install_venv(package_dir, settings, verbose=False): """ Install requirements into the virtual environment. :param package_dir: the full path to the package directory :param settings: the settings dict (incluing the venv_dir name) :param verbose: (optional) be more verbose if set to True, defaults to False """ venv_dir = os.path.join(package_dir, settings.get('venv_dir')) if not os.path.exists(venv_dir): echo_failure( 'virtual environment directory `{}` does not exists, nothing to install' .format(venv_dir)) sys.exit(1) if not os.path.exists( os.path.join(package_dir, settings.get('requirements_filename'))): echo_failure( 'could not find requirements.txt to install, check if `{}` exists or update settings' .format(settings.get('requirements_filename'))) sys.exit(1) install_command = [ sys.executable, '-m', 'pip', 'install', '-r', os.path.join(package_dir, settings.get('requirements_filename')) ] if not verbose: # no cov install_command.append('-qqq') with venv(venv_dir): echo_info( 'Installing requirements from `{}` into the virtual environment `{}`' .format(settings.get('requirements_filename'), settings.get('venv_dir'))) result = None if six.PY3: result = subprocess.run(install_command, shell=NEED_SUBPROCESS_SHELL) return result.returncode elif six.PY2: result = subprocess.check_output(install_command, shell=NEED_SUBPROCESS_SHELL) return result and 0 or -1 if result: echo_success(str(result)) return result.returncode
def sign_package(package_dir, settings, options=None, verbose=False): """ Sign the package with a GPG/PGP key. :param package_dir: directory fullpath of the package :param settings: settings object :param options: commandline options dictionary passed down. :param verbose: be verbose (or not) :return: None """ gpg = get_gpg() if options.get('sign_keyid') is None: tabulate_keys(gpg, explain=True) options['sign_keyid'] = click.prompt( "Provide Key (Name, Comment, Email, Fingerprint) to sign package with", default=settings.get('email')) if options.get('sign_passphrase') is None: options['sign_passphrase'] = click.prompt("Provide Passphrase", hide_input=True) echo_info('Signing package contents') with open( os.path.join( package_dir, settings.get('artifacts_filename', ARTIFACTS_FILENAME)), 'rb') as fd: results = gpg.sign_file(fd, keyid=options.get('sign_keyid'), passphrase=options.get('sign_passphrase'), detach=True, output=settings.get('artifacts_sig_filename', ARTIFACTS_SIG_FILENAME)) pprint(results.__dict__) if results and results.status is not None: echo_info("Signed package contents: {}".format(results.status)) else: failure_text = results.stderr.split("\n")[-2] echo_failure( "Could not sign the package contents: '{}'".format(failure_text)) sys.exit(1) if verbose: echo_success('Successfully signed the package contents.') verify_signature(package_dir, ARTIFACTS_FILENAME, ARTIFACTS_SIG_FILENAME) verify_artifacts_hashes(package_dir, ARTIFACTS_FILENAME)
def verify_artifacts_hashes(package_dir, artifacts_filename): """ Check the hashes of the artifacts in the package. :param package_dir: directory fullpath of the package :param artifacts_filename: filename of the artifacts file :return: """ artifacts_fp = os.path.join(package_dir, artifacts_filename) if not os.path.exists(artifacts_fp): echo_failure( "Artifacts file does not exist: '{}'".format(artifacts_filename)) sys.exit(1) with open(artifacts_fp, 'r') as fd: artifacts = fd.readlines() # process the file contents # A line is "README.md,sha256=d831....ccf79a,336" # ^filename ^algo ^hash ^size in bytes fails = [] for af in artifacts: # noinspection PyShadowingBuiltins,PyShadowingBuiltins filename, hash, orig_size = af.split(',') algorithm, orig_hash = hash.split('=') fp = os.path.join(package_dir, filename) if os.path.exists(fp): found_hash = hash_of_file(fp, algorithm) found_size = os.stat(fp).st_size if found_hash != orig_hash.strip() or found_size != int( orig_size.strip()): fails.append( "File '{}' is changed in the package.".format(filename)) fails.append( "File '{}' original checksum: '{}', found: '{}'".format( filename, orig_hash, found_hash)) fails.append("File '{}' original size: {}, found: {}".format( filename, orig_size, found_size)) else: fails.append("File '{}' does not exist".format(filename)) if fails: echo_failure( 'The package has been changed after building the package.') for fail in fails: print(fail) sys.exit(1) else: echo_info("Package contents succesfully verified.")
def load_settings(lazy=False, package_dir=None, settings_filename=None): """ Load settings from disk. :param lazy: (optional) does lazy loading (default to False) :param package_dir: (optional) loads the settings from a package dir :param settings_filename: (optional) pathname of the file where the settings are stored :return: settings dictionary """ settings_filepath = get_settings_filepath(package_dir, settings_filename) if lazy and not os.path.exists(settings_filepath): return {} elif not os.path.exists(settings_filepath): echo_failure('Could not find a settingsfile in path: {}'.format( settings_filepath)) sys.exit(404) else: with open(settings_filepath, 'r') as f: return json.loads(f.read(), object_pairs_hook=OrderedDict)
def prune(package, **options): """Remove a project's build artifacts.""" package_name = package or get_package_name() or click.prompt('Provide package name') package_dir = get_package_dir(package_name) settings = load_settings(package_name) # ensure build directory is there build_dir = settings.get('build_dir', 'dist') build_path = os.path.join(package_dir, build_dir) if os.path.exists(build_path): if options.get('force') or click.confirm( "Do you want to prune build artifacts for package '{}'?".format(package_name)): remove_path(build_path) if os.path.exists(build_path): echo_failure('Something went wrong pruning pacakage `{}`'.format(package_name)) else: echo_warning('Package `{}` will not be pruned'.format(package_name)) else: echo_failure('Package `{}` does not exist'.format(package_name))
def _do_export_key(gpg, options): """Export public key.""" echo_info("Exporting public key") if options.get('keyid') is None: _do_list(gpg=gpg) options['keyid'] = click.prompt( "Provide KeyId (name, comment, email, fingerprint) of the key to export" ) result = gpg.export_keys(keyids=[options.get('keyid')], secret=False, armor=True) if result is not None: with open(options.get('do_export_key'), 'w') as fd: fd.write(result) echo_success("Sucessfully written public key to '{}'".format( options.get('do_export_key'))) sys.exit(0) echo_failure("Could not export key") sys.exit(1)
def purge(package, **options): """ Purge and clean a package directory structure. :param package: Name of the kecpkg package :param options: :return: """ package_name = package or click.prompt('Provide package name') package_dir = get_package_dir(package_name) if os.path.exists(package_dir): if options.get('force') or click.confirm( "Do you want to purge and completely remove '{}'?".format(package_name)): remove_path(package_dir) if not os.path.exists(package_dir): echo_success('Package `{}` is purged and removed from disk'.format(package_name)) else: echo_failure('Something went wrong pruning pacakage `{}`'.format(package_name)) else: echo_warning('Package `{}` will not be purged'.format(package_name)) else: echo_failure('Package `{}` does not exist'.format(package_name))
def _do_import(gpg, options): echo_info("Importing secret key into KECPKG keyring from '{}'".format( options.get('do_import'))) result = gpg.import_keys( open(os.path.abspath(options.get('do_import')), 'rb').read()) # pprint(result.__dict__) if result and result.sec_imported: echo_success( "Succesfully imported secret key into the KECPKG keystore") _do_list(gpg=gpg) sys.exit(0) elif result and result.unchanged: echo_failure( "Did not import the secret key into the KECPKG keystore. The key was already " "in place and was unchanged") _do_list(gpg=gpg) sys.exit(1) echo_failure( "Did not import a secret key into the KECPKG keystore. Is something wrong " "with the file: '{}'? Are you sure it is a ASCII file containing a " "private key block?".format(options.get('do_import'))) sys.exit(1)
def new(package=None, **options): """ Create a new package directory structure. <pkg dir> +-- venv | +-- ... <the virtualenvironment> +-- README.md +-- requirements.txt +-- script.py +-- package-info.json +-- .gitignore +-- .kecpkg-settings.json """ settings = load_settings(lazy=True, settings_filename=options.get('settings_filename')) if not settings: settings = copy_default_settings() package_root_dir = os.getcwd() # set the package name, clean an normalise using snake_case package_name = package or click.prompt("Package name") package_name = normalise_name(package_name) # save to settings settings['package_name'] = package_name package_dir = os.path.join(package_root_dir, package_name) if os.path.exists(package_dir): echo_failure("Directory '{}' already exists.".format(package_dir)) sys.exit(1) if not package: settings['version'] = click.prompt('Version', default=settings.get('version', '0.0.1')) settings['description'] = click.prompt('Description', default='') settings['name'] = click.prompt('Author', default=settings.get('name', os.environ.get('USER', ''))) settings['email'] = click.prompt('Author\'s email', default=settings.get('email', '')) settings['python_version'] = click.prompt('Python version (choose from: {})'.format(settings.get('pyversions')), default='3.5') settings['exclude_paths'] = click.prompt("Exclude additional paths from kecpkg (eg. 'data, input')", default=settings.get('exclude_paths', ''), value_proc=process_additional_exclude_paths) if options.get('script'): script_base = normalise_name(options.get('script').replace('.py', '')) echo_info('Setting the script to `{}`'.format(script_base)) settings['entrypoint_script'] = script_base if options.get('venv'): settings['venv_dir'] = normalise_name(options.get('venv')) echo_info("Creating package structure") create_package(package_dir, settings=settings) if not options.get('no_venv'): echo_info("Creating virtual environment") create_venv(package_dir, settings, pypath=None, use_global=options.get('global_packages'), verbose=options.get('verbose')) pip_install_venv(package_dir, settings, verbose=options.get('verbose')) else: settings['venv_dir'] = None # save the settings (in the package_dir) save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) echo_success('Package `{package_name}` created in `{package_dir}`'.format(package_name=package_name, package_dir=package_dir))
def _do_create_key(gpg, options): echo_info( "Will create a secret key and store it into the KECPKG keyring.") package_dir = get_package_dir(package_name=package, fail=False) settings = DEFAULT_SETTINGS if package_dir is not None: package_name = os.path.basename(package_dir) echo_info('Package `{}` has been selected'.format(package_name)) settings = load_settings( package_dir=package_dir, settings_filename=options.get('settings_filename')) key_info = { 'name_real': click.prompt("Name", default=settings.get('name')), 'name_comment': click.prompt("Comment", default="KECPKG SIGNING KEY"), 'name_email': click.prompt("Email", default=settings.get('email')), 'expire_date': click.prompt("Expiration in months", default=12, value_proc=lambda i: "{}m".format(i)), 'key_type': 'RSA', 'key_length': 4096, 'key_usage': '', 'subkey_type': 'RSA', 'subkey_length': 4096, 'subkey_usage': 'encrypt,sign,auth', 'passphrase': '' } passphrase = click.prompt("Passphrase", hide_input=True) passphrase_confirmed = click.prompt("Confirm passphrase", hide_input=True) if passphrase == passphrase_confirmed: key_info['passphrase'] = passphrase else: raise ValueError("The passphrases did not match.") echo_info( "Creating the secret key '{name_real} ({name_comment}) <{name_email}>'" .format(**key_info)) echo_info( "Please move around mouse or generate other activity to introduce sufficient entropy. " "This might take a minute...") result = gpg.gen_key(gpg.gen_key_input(**key_info)) pprint(result.__dict__) if result and result.stderr.find('KEY_CREATED'): echo_success("The key is succesfully created") _do_list(gpg=gpg) sys.exit(0) echo_failure("Could not generate the key due to an error: '{}'".format( result.stderr)) sys.exit(1)
def upload(package=None, url=None, username=None, password=None, token=None, scope=None, scope_id=None, kecpkg=None, **options): """ Upload built kecpkg to KE-chain. If no options are provided, the interactive mode is triggered. """ package_name = package or get_package_name() or click.prompt( 'Package name') settings = load_settings( package_dir=get_package_dir(package_name), settings_filename=options.get('settings_filename')) if not url or not ((username and password) or token): url = click.prompt('Url (incl http(s)://)', default=settings.get('url') or url) username = click.prompt('Username', default=settings.get('username') or username) password = click.prompt('Password', hide_input=True) # set the interactive world to True for continuation sake options['interactive'] = True elif not options.get('interactive'): url = url or settings.get('url') username = username or settings.get('username') token = token or settings.get('token') scope_id = scope_id or settings.get('scope_id') client = Client(url) client.login(username=username, password=password, token=token) # scope finder if not scope_id and settings.get('scope_id') and \ click.confirm("Do you wish to use the stored `scope_id` in settings: `{}`".format( settings.get('scope_id')), default=True): scope_id = settings.get('scope_id') if not scope_id: scopes = client.scopes() scope_matcher = [ dict(number=i, scope_id=scope.id, scope=scope.name) for i, scope in zip(range(1, len(scopes)), scopes) ] # nice UI echo_info('Choose from following scopes:') for match_dict in scope_matcher: echo_info( "{number} | {scope_id:.8} | {scope}".format(**match_dict)) scope_match = None while not scope_match: scope_guess = click.prompt('Row number, part of Id or Scope') scope_match = validate_scopes(scope_guess, scope_matcher) echo_success( "Scope selected: '{scope}' ({scope_id})".format(**scope_match)) scope_id = scope_match['scope_id'] scope_to_upload = get_project(url, username, password, token, scope_id=scope_id) # service reupload service_id = options.get('service_id') or settings.get('service_id') if options.get('reupload') and not service_id: echo_failure('Please provide a service id to reupload to.') elif service_id and not options.get('reupload') and options.get( 'interactive'): if click.confirm( "Do you wish to *replace* the previously uploaded service: `{}`" .format(service_id), default=True): service_id = service_id else: service_id = None # store to settings if options.get('store'): settings.update( dict(url=url, username=username, scope_id=str(scope_id))) if service_id: settings['service_id'] = str(service_id) save_settings(settings, settings_filename=options.get('settings_filename')) # do upload build_path = os.path.join(get_package_dir(package_name), settings.get('build_dir')) if not os.path.exists(build_path): echo_failure('Cannot find build path, please do `kecpkg build` first') sys.exit(400) upload_package(scope_to_upload, build_path, kecpkg, service_id=service_id, settings=settings, store_settings=options.get('store'), settings_filename=options.get('settings_filename'))
def upload_package(scope, build_path=None, kecpkg_path=None, service_id=None, settings=None, store_settings=True, settings_filename=None): """ Upload the package from build_path to the right scope, create a new KE-chain SIM service. :param scope: Scope object (pykechain) :param build_path: path to the build directory in which the to-be uploaded script resides :param kecpkg_path: path to the kecpkg file to upload (no need to provide build_path) :param service_id: UUID of the service to upload to :param settings: settings of the package :param store_settings: store the settings after update (eg service_id after upload) :param settings_filename: pathname of the file where the settings are stored :return: None """ # if not (kecpkg_path and not build_path) or not (build_path and not kecpkg_path): # echo_failure("You should provide a build path or a kecpkg path") # sys.exit(404) if kecpkg_path and os.path.exists(kecpkg_path): kecpkg_path = kecpkg_path else: built_kecpkgs = os.listdir(build_path) if not kecpkg_path and len(built_kecpkgs) > 1 and settings.get( 'version'): built_kecpkgs = [ f for f in built_kecpkgs if settings.get('version') in f ] if not kecpkg_path and len(built_kecpkgs) == 1: kecpkg_path = os.path.join(build_path, built_kecpkgs[0]) else: echo_info('Provide correct filename to upload') echo_info('\n'.join(os.listdir(build_path))) kecpkg_filename = click.prompt('Filename') kecpkg_path = os.path.join(build_path, kecpkg_filename) if kecpkg_path and os.path.exists(kecpkg_path): # ready to upload echo_info('Ready to upload `{}`'.format(os.path.basename(kecpkg_path))) else: echo_failure('Unable to locate kecpkg to upload') sys.exit(404) # get meta and prepare 2 stage submission # 1. fill service information # 2. do upload if service_id: service = scope.service(pk=service_id) service.upload(kecpkg_path) service.edit(name=settings.get('package_name'), description=settings.get('description', ''), script_version=settings.get('version', '')) else: # Create new service in KE-chain service = scope.create_service( name=settings.get('package_name'), description=settings.get('description', ''), version=settings.get('version', ''), service_type='PYTHON SCRIPT', environment_version=settings.get('python_version'), pkg_path=kecpkg_path) # Wrap up party! echo_success("kecpkg `{}` successfully uploaded to KE-chain.".format( os.path.basename(kecpkg_path))) # noinspection PyProtectedMember success_url = "{api_root}/#scopes/{scope_id}/scripts/{service_id}".format( api_root=scope._client.api_root, scope_id=scope.id, service_id=service.id) echo_success( "To view the newly created service, go to: `{}`".format(success_url)) # update settings if store_settings: settings['service_id'] = str(service.id) from datetime import datetime settings['last_upload'] = str(datetime.now().isoformat()) save_settings(settings, settings_filename=settings_filename)