def build(package=None, **options): """Build the package and create a kecpkg file.""" echo_info('Locating package ``'.format(package)) package_dir = get_package_dir(package_name=package) package_name = os.path.basename(package_dir) echo_info('Package `{}` has been selected'.format(package_name)) settings = load_settings( package_dir=package_dir, settings_filename=options.get('settings_filename')) # ensure build directory is there build_dir = settings.get('build_dir', 'dist') build_path = os.path.join(package_dir, build_dir) if options.get('update_package_info'): render_package_info(settings, package_dir=package_dir, backup=True) if options.get('clean_first'): remove_path(build_path) ensure_dir_exists(build_path) # do package building build_package(package_dir, build_path, settings, options=options, verbose=options.get('verbose')) echo_success('Complete')
def _do_delete_key(gpg, options): echo_info( "Deleting private key with ID '{}' from the KECPKG keyring".format( options.get('do_delete_key'))) # custom call to gpg using --delete-secret-and-public-key result = gpg.result_map['delete'](gpg) # noinspection PyProtectedMember p = gpg._open_subprocess([ '--yes', '--delete-secret-and-public-key', options.get('do_delete_key') ]) # noinspection PyProtectedMember gpg._collect_output(p, result, stdin=p.stdin) # result = gpg.delete_keys(fingerprints=options.get('do_delete_key'), # secret=True, # passphrase=options.get('sign_passphrase')) # pprint(result.__dict__) if result and result.stderr.find("failed") < 0: echo_success("Succesfully deleted key") _do_list(gpg=gpg) sys.exit(0) echo_failure("Could not delete key.") sys.exit(1)
def _do_clear(options): echo_info("Clearing all keys from the KECPKG keyring") if not options.get('do_yes'): options['do_yes'] = click.confirm( "Are you sure you want to clear the KECPKG keyring?", default=False) if options.get('do_yes'): remove_path(GNUPG_KECPKG_HOME) echo_success("Completed") sys.exit(0) else: echo_failure("Not removing the KECPKG keyring") sys.exit(1)
def pip_install_venv(package_dir, settings, verbose=False): """ Install requirements into the virtual environment. :param package_dir: the full path to the package directory :param settings: the settings dict (incluing the venv_dir name) :param verbose: (optional) be more verbose if set to True, defaults to False """ venv_dir = os.path.join(package_dir, settings.get('venv_dir')) if not os.path.exists(venv_dir): echo_failure( 'virtual environment directory `{}` does not exists, nothing to install' .format(venv_dir)) sys.exit(1) if not os.path.exists( os.path.join(package_dir, settings.get('requirements_filename'))): echo_failure( 'could not find requirements.txt to install, check if `{}` exists or update settings' .format(settings.get('requirements_filename'))) sys.exit(1) install_command = [ sys.executable, '-m', 'pip', 'install', '-r', os.path.join(package_dir, settings.get('requirements_filename')) ] if not verbose: # no cov install_command.append('-qqq') with venv(venv_dir): echo_info( 'Installing requirements from `{}` into the virtual environment `{}`' .format(settings.get('requirements_filename'), settings.get('venv_dir'))) result = None if six.PY3: result = subprocess.run(install_command, shell=NEED_SUBPROCESS_SHELL) return result.returncode elif six.PY2: result = subprocess.check_output(install_command, shell=NEED_SUBPROCESS_SHELL) return result and 0 or -1 if result: echo_success(str(result)) return result.returncode
def sign_package(package_dir, settings, options=None, verbose=False): """ Sign the package with a GPG/PGP key. :param package_dir: directory fullpath of the package :param settings: settings object :param options: commandline options dictionary passed down. :param verbose: be verbose (or not) :return: None """ gpg = get_gpg() if options.get('sign_keyid') is None: tabulate_keys(gpg, explain=True) options['sign_keyid'] = click.prompt( "Provide Key (Name, Comment, Email, Fingerprint) to sign package with", default=settings.get('email')) if options.get('sign_passphrase') is None: options['sign_passphrase'] = click.prompt("Provide Passphrase", hide_input=True) echo_info('Signing package contents') with open( os.path.join( package_dir, settings.get('artifacts_filename', ARTIFACTS_FILENAME)), 'rb') as fd: results = gpg.sign_file(fd, keyid=options.get('sign_keyid'), passphrase=options.get('sign_passphrase'), detach=True, output=settings.get('artifacts_sig_filename', ARTIFACTS_SIG_FILENAME)) pprint(results.__dict__) if results and results.status is not None: echo_info("Signed package contents: {}".format(results.status)) else: failure_text = results.stderr.split("\n")[-2] echo_failure( "Could not sign the package contents: '{}'".format(failure_text)) sys.exit(1) if verbose: echo_success('Successfully signed the package contents.') verify_signature(package_dir, ARTIFACTS_FILENAME, ARTIFACTS_SIG_FILENAME) verify_artifacts_hashes(package_dir, ARTIFACTS_FILENAME)
def _do_export_key(gpg, options): """Export public key.""" echo_info("Exporting public key") if options.get('keyid') is None: _do_list(gpg=gpg) options['keyid'] = click.prompt( "Provide KeyId (name, comment, email, fingerprint) of the key to export" ) result = gpg.export_keys(keyids=[options.get('keyid')], secret=False, armor=True) if result is not None: with open(options.get('do_export_key'), 'w') as fd: fd.write(result) echo_success("Sucessfully written public key to '{}'".format( options.get('do_export_key'))) sys.exit(0) echo_failure("Could not export key") sys.exit(1)
def purge(package, **options): """ Purge and clean a package directory structure. :param package: Name of the kecpkg package :param options: :return: """ package_name = package or click.prompt('Provide package name') package_dir = get_package_dir(package_name) if os.path.exists(package_dir): if options.get('force') or click.confirm( "Do you want to purge and completely remove '{}'?".format(package_name)): remove_path(package_dir) if not os.path.exists(package_dir): echo_success('Package `{}` is purged and removed from disk'.format(package_name)) else: echo_failure('Something went wrong pruning pacakage `{}`'.format(package_name)) else: echo_warning('Package `{}` will not be purged'.format(package_name)) else: echo_failure('Package `{}` does not exist'.format(package_name))
def _do_import(gpg, options): echo_info("Importing secret key into KECPKG keyring from '{}'".format( options.get('do_import'))) result = gpg.import_keys( open(os.path.abspath(options.get('do_import')), 'rb').read()) # pprint(result.__dict__) if result and result.sec_imported: echo_success( "Succesfully imported secret key into the KECPKG keystore") _do_list(gpg=gpg) sys.exit(0) elif result and result.unchanged: echo_failure( "Did not import the secret key into the KECPKG keystore. The key was already " "in place and was unchanged") _do_list(gpg=gpg) sys.exit(1) echo_failure( "Did not import a secret key into the KECPKG keystore. Is something wrong " "with the file: '{}'? Are you sure it is a ASCII file containing a " "private key block?".format(options.get('do_import'))) sys.exit(1)
def new(package=None, **options): """ Create a new package directory structure. <pkg dir> +-- venv | +-- ... <the virtualenvironment> +-- README.md +-- requirements.txt +-- script.py +-- package-info.json +-- .gitignore +-- .kecpkg-settings.json """ settings = load_settings(lazy=True, settings_filename=options.get('settings_filename')) if not settings: settings = copy_default_settings() package_root_dir = os.getcwd() # set the package name, clean an normalise using snake_case package_name = package or click.prompt("Package name") package_name = normalise_name(package_name) # save to settings settings['package_name'] = package_name package_dir = os.path.join(package_root_dir, package_name) if os.path.exists(package_dir): echo_failure("Directory '{}' already exists.".format(package_dir)) sys.exit(1) if not package: settings['version'] = click.prompt('Version', default=settings.get('version', '0.0.1')) settings['description'] = click.prompt('Description', default='') settings['name'] = click.prompt('Author', default=settings.get('name', os.environ.get('USER', ''))) settings['email'] = click.prompt('Author\'s email', default=settings.get('email', '')) settings['python_version'] = click.prompt('Python version (choose from: {})'.format(settings.get('pyversions')), default='3.5') settings['exclude_paths'] = click.prompt("Exclude additional paths from kecpkg (eg. 'data, input')", default=settings.get('exclude_paths', ''), value_proc=process_additional_exclude_paths) if options.get('script'): script_base = normalise_name(options.get('script').replace('.py', '')) echo_info('Setting the script to `{}`'.format(script_base)) settings['entrypoint_script'] = script_base if options.get('venv'): settings['venv_dir'] = normalise_name(options.get('venv')) echo_info("Creating package structure") create_package(package_dir, settings=settings) if not options.get('no_venv'): echo_info("Creating virtual environment") create_venv(package_dir, settings, pypath=None, use_global=options.get('global_packages'), verbose=options.get('verbose')) pip_install_venv(package_dir, settings, verbose=options.get('verbose')) else: settings['venv_dir'] = None # save the settings (in the package_dir) save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) echo_success('Package `{package_name}` created in `{package_dir}`'.format(package_name=package_name, package_dir=package_dir))
def _do_create_key(gpg, options): echo_info( "Will create a secret key and store it into the KECPKG keyring.") package_dir = get_package_dir(package_name=package, fail=False) settings = DEFAULT_SETTINGS if package_dir is not None: package_name = os.path.basename(package_dir) echo_info('Package `{}` has been selected'.format(package_name)) settings = load_settings( package_dir=package_dir, settings_filename=options.get('settings_filename')) key_info = { 'name_real': click.prompt("Name", default=settings.get('name')), 'name_comment': click.prompt("Comment", default="KECPKG SIGNING KEY"), 'name_email': click.prompt("Email", default=settings.get('email')), 'expire_date': click.prompt("Expiration in months", default=12, value_proc=lambda i: "{}m".format(i)), 'key_type': 'RSA', 'key_length': 4096, 'key_usage': '', 'subkey_type': 'RSA', 'subkey_length': 4096, 'subkey_usage': 'encrypt,sign,auth', 'passphrase': '' } passphrase = click.prompt("Passphrase", hide_input=True) passphrase_confirmed = click.prompt("Confirm passphrase", hide_input=True) if passphrase == passphrase_confirmed: key_info['passphrase'] = passphrase else: raise ValueError("The passphrases did not match.") echo_info( "Creating the secret key '{name_real} ({name_comment}) <{name_email}>'" .format(**key_info)) echo_info( "Please move around mouse or generate other activity to introduce sufficient entropy. " "This might take a minute...") result = gpg.gen_key(gpg.gen_key_input(**key_info)) pprint(result.__dict__) if result and result.stderr.find('KEY_CREATED'): echo_success("The key is succesfully created") _do_list(gpg=gpg) sys.exit(0) echo_failure("Could not generate the key due to an error: '{}'".format( result.stderr)) sys.exit(1)
def config(package, **options): """Manage the configuration (or settings) of the package. The various settings in the .kecpkg-settings.json file are: package_name: name of the package version: version number of the package description: longer description of the package name: name of the author of the package email: email of the author of the package python_version: python version on which the kecpkg will run, corresponds with an executable KE-crunch environment entrypoint_script: the name of the script that will be executed first entrypoint_func: function name inside the script that will be executed. Ensure that it takes *args, **kwargs. venv_dir: python virtual environment directory in the development environment requirements_filename: name of the requirements file with list of package that will be installed before running build_dir: directory where the built kecpkg will be stored exclude_paths: list of paths that will be excluded from the package, next to the build in excludes url: url where the package will be uploaded token: token of the user under which the package is uploaded scope_id: identification of the scope under which the package is uploaded service_id: identification under which the package is re-uploaded (or recently uploaded) last_upload: date and time of the last upload """ echo_info('Locating package ``'.format(package)) package_dir = get_package_dir(package_name=package) package_name = os.path.basename(package_dir) echo_info('Package `{}` has been selected'.format(package_name)) if options.get('init'): if os.path.exists(os.path.join(package_dir, options.get('settings_filename'))) and \ click.confirm('Are you sure you want to overwrite the current settingsfile ' '(old settings will be a backup)?'): copy_path( os.path.join(package_dir, options.get('settings_filename')), os.path.join( package_dir, "{}-backup".format(options.get('settings_filename')))) echo_info('Creating new settingsfile') settings = copy_default_settings() settings['package_name'] = package_name save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) settings = load_settings(package_dir=package_dir) if options.get('interactive'): settings['version'] = click.prompt('Version', default=settings.get( 'version', '0.0.1')) settings['description'] = click.prompt('Description', default=settings.get( 'description', '')) settings['name'] = click.prompt('Author', default=settings.get( 'name', os.environ.get('USER', ''))) settings['email'] = click.prompt('Author\'s email', default=settings.get('email', '')) settings['python_version'] = click.prompt( 'Python version (choose from: {})'.format( settings.get('pyversions')), default='3.5') settings['exclude_paths'] = click.prompt( "Exclude additional paths from kecpkg (eg. 'data, input')", default=settings.get('exclude_paths', ''), value_proc=process_additional_exclude_paths) save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) if options.get('set_key'): k, v = options.get('set_key') if options.get('verbose'): echo_info("Set the key '{}' to value '{}'".format(k, v)) settings[k] = v save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) if options.get('get_key'): echo_info( tabulate([ (options.get('get_key'), settings.get(options.get('get_key'))) ], headers=("key", "value"))) return if options.get('verbose'): echo_info(tabulate(settings.items(), headers=("key", "value"))) if not options.get('interactive'): echo_success('Settings file identified and correct')
def upload(package=None, url=None, username=None, password=None, token=None, scope=None, scope_id=None, kecpkg=None, **options): """ Upload built kecpkg to KE-chain. If no options are provided, the interactive mode is triggered. """ package_name = package or get_package_name() or click.prompt( 'Package name') settings = load_settings( package_dir=get_package_dir(package_name), settings_filename=options.get('settings_filename')) if not url or not ((username and password) or token): url = click.prompt('Url (incl http(s)://)', default=settings.get('url') or url) username = click.prompt('Username', default=settings.get('username') or username) password = click.prompt('Password', hide_input=True) # set the interactive world to True for continuation sake options['interactive'] = True elif not options.get('interactive'): url = url or settings.get('url') username = username or settings.get('username') token = token or settings.get('token') scope_id = scope_id or settings.get('scope_id') client = Client(url) client.login(username=username, password=password, token=token) # scope finder if not scope_id and settings.get('scope_id') and \ click.confirm("Do you wish to use the stored `scope_id` in settings: `{}`".format( settings.get('scope_id')), default=True): scope_id = settings.get('scope_id') if not scope_id: scopes = client.scopes() scope_matcher = [ dict(number=i, scope_id=scope.id, scope=scope.name) for i, scope in zip(range(1, len(scopes)), scopes) ] # nice UI echo_info('Choose from following scopes:') for match_dict in scope_matcher: echo_info( "{number} | {scope_id:.8} | {scope}".format(**match_dict)) scope_match = None while not scope_match: scope_guess = click.prompt('Row number, part of Id or Scope') scope_match = validate_scopes(scope_guess, scope_matcher) echo_success( "Scope selected: '{scope}' ({scope_id})".format(**scope_match)) scope_id = scope_match['scope_id'] scope_to_upload = get_project(url, username, password, token, scope_id=scope_id) # service reupload service_id = options.get('service_id') or settings.get('service_id') if options.get('reupload') and not service_id: echo_failure('Please provide a service id to reupload to.') elif service_id and not options.get('reupload') and options.get( 'interactive'): if click.confirm( "Do you wish to *replace* the previously uploaded service: `{}`" .format(service_id), default=True): service_id = service_id else: service_id = None # store to settings if options.get('store'): settings.update( dict(url=url, username=username, scope_id=str(scope_id))) if service_id: settings['service_id'] = str(service_id) save_settings(settings, settings_filename=options.get('settings_filename')) # do upload build_path = os.path.join(get_package_dir(package_name), settings.get('build_dir')) if not os.path.exists(build_path): echo_failure('Cannot find build path, please do `kecpkg build` first') sys.exit(400) upload_package(scope_to_upload, build_path, kecpkg, service_id=service_id, settings=settings, store_settings=options.get('store'), settings_filename=options.get('settings_filename'))
def upload_package(scope, build_path=None, kecpkg_path=None, service_id=None, settings=None, store_settings=True, settings_filename=None): """ Upload the package from build_path to the right scope, create a new KE-chain SIM service. :param scope: Scope object (pykechain) :param build_path: path to the build directory in which the to-be uploaded script resides :param kecpkg_path: path to the kecpkg file to upload (no need to provide build_path) :param service_id: UUID of the service to upload to :param settings: settings of the package :param store_settings: store the settings after update (eg service_id after upload) :param settings_filename: pathname of the file where the settings are stored :return: None """ # if not (kecpkg_path and not build_path) or not (build_path and not kecpkg_path): # echo_failure("You should provide a build path or a kecpkg path") # sys.exit(404) if kecpkg_path and os.path.exists(kecpkg_path): kecpkg_path = kecpkg_path else: built_kecpkgs = os.listdir(build_path) if not kecpkg_path and len(built_kecpkgs) > 1 and settings.get( 'version'): built_kecpkgs = [ f for f in built_kecpkgs if settings.get('version') in f ] if not kecpkg_path and len(built_kecpkgs) == 1: kecpkg_path = os.path.join(build_path, built_kecpkgs[0]) else: echo_info('Provide correct filename to upload') echo_info('\n'.join(os.listdir(build_path))) kecpkg_filename = click.prompt('Filename') kecpkg_path = os.path.join(build_path, kecpkg_filename) if kecpkg_path and os.path.exists(kecpkg_path): # ready to upload echo_info('Ready to upload `{}`'.format(os.path.basename(kecpkg_path))) else: echo_failure('Unable to locate kecpkg to upload') sys.exit(404) # get meta and prepare 2 stage submission # 1. fill service information # 2. do upload if service_id: service = scope.service(pk=service_id) service.upload(kecpkg_path) service.edit(name=settings.get('package_name'), description=settings.get('description', ''), script_version=settings.get('version', '')) else: # Create new service in KE-chain service = scope.create_service( name=settings.get('package_name'), description=settings.get('description', ''), version=settings.get('version', ''), service_type='PYTHON SCRIPT', environment_version=settings.get('python_version'), pkg_path=kecpkg_path) # Wrap up party! echo_success("kecpkg `{}` successfully uploaded to KE-chain.".format( os.path.basename(kecpkg_path))) # noinspection PyProtectedMember success_url = "{api_root}/#scopes/{scope_id}/scripts/{service_id}".format( api_root=scope._client.api_root, scope_id=scope.id, service_id=service.id) echo_success( "To view the newly created service, go to: `{}`".format(success_url)) # update settings if store_settings: settings['service_id'] = str(service.id) from datetime import datetime settings['last_upload'] = str(datetime.now().isoformat()) save_settings(settings, settings_filename=settings_filename)