def test_build_with_alternate_config(self): pkgname = 'new_pkg' alt_settings = 'alt-settings.json' with temp_chdir() as d: runner = CliRunner() result = runner.invoke(kecpkg, ['new', pkgname, '--no-venv']) package_dir = get_package_dir(pkgname) self.assertTrue(os.path.exists(package_dir)) # set alternative settings path settings = copy_default_settings() settings["package_name"] = pkgname save_settings(settings, package_dir=package_dir, settings_filename=alt_settings) os.chdir(package_dir) result = runner.invoke( kecpkg, ['build', pkgname, '--config', alt_settings]) self.assertEqual( result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format( result.output)) self.assertExists(os.path.join(package_dir, 'dist')) dist_dir_contents = os.listdir(os.path.join(package_dir, 'dist')) self.assertTrue(len(dist_dir_contents), 1) self.assertTrue( pkgname in dist_dir_contents[0], "the name of the pkg `{}` should be in the name of " "the built kecpkg `{}`".format(pkgname, dist_dir_contents[0]))
def test_build_with_extra_ignores(self): pkgname = 'new_pkg' with temp_chdir() as d: runner = CliRunner() result = runner.invoke(kecpkg, ['new', pkgname, '--no-venv']) package_dir = get_package_dir(pkgname) self.assertTrue(os.path.exists(package_dir)) ensure_dir_exists(os.path.join(package_dir, 'data')) # add additional files (to exclude for building later) touch_file(os.path.join(package_dir, 'data', 'somefile.txt')) touch_file( os.path.join(package_dir, 'local_extra_file.someext.txt')) os.chdir(package_dir) # check if those files exists package_dir_contents = os.listdir(os.path.join(package_dir)) self.assertTrue( 'local_extra_file.someext.txt' in package_dir_contents) self.assertTrue('data' in package_dir_contents) # set exclude_paths in settings settings = copy_default_settings() settings["exclude_paths"] = ["data", "local_extra_file.*"] save_settings(settings, package_dir=package_dir) # run the builder result = runner.invoke(kecpkg, ['build', pkgname, '--verbose']) self.assertEqual( result.exit_code, 0, "Results of the run were: \n---\n{}\n---".format( result.output)) self.assertExists(os.path.join(package_dir, 'dist')) # check the zip such that the extra files are not packaged dist_list = os.listdir(os.path.join(package_dir, 'dist')) zipfile = ZipFile(os.path.join(package_dir, 'dist', dist_list[0]), 'r') contents = zipfile.namelist() self.assertFalse('local_extra_file.someext.txt' in contents) self.assertFalse('data' in contents)
def new(package=None, **options): """ Create a new package directory structure. <pkg dir> +-- venv | +-- ... <the virtualenvironment> +-- README.md +-- requirements.txt +-- script.py +-- package-info.json +-- .gitignore +-- .kecpkg-settings.json """ settings = load_settings(lazy=True, settings_filename=options.get('settings_filename')) if not settings: settings = copy_default_settings() package_root_dir = os.getcwd() # set the package name, clean an normalise using snake_case package_name = package or click.prompt("Package name") package_name = normalise_name(package_name) # save to settings settings['package_name'] = package_name package_dir = os.path.join(package_root_dir, package_name) if os.path.exists(package_dir): echo_failure("Directory '{}' already exists.".format(package_dir)) sys.exit(1) if not package: settings['version'] = click.prompt('Version', default=settings.get('version', '0.0.1')) settings['description'] = click.prompt('Description', default='') settings['name'] = click.prompt('Author', default=settings.get('name', os.environ.get('USER', ''))) settings['email'] = click.prompt('Author\'s email', default=settings.get('email', '')) settings['python_version'] = click.prompt('Python version (choose from: {})'.format(settings.get('pyversions')), default='3.5') settings['exclude_paths'] = click.prompt("Exclude additional paths from kecpkg (eg. 'data, input')", default=settings.get('exclude_paths', ''), value_proc=process_additional_exclude_paths) if options.get('script'): script_base = normalise_name(options.get('script').replace('.py', '')) echo_info('Setting the script to `{}`'.format(script_base)) settings['entrypoint_script'] = script_base if options.get('venv'): settings['venv_dir'] = normalise_name(options.get('venv')) echo_info("Creating package structure") create_package(package_dir, settings=settings) if not options.get('no_venv'): echo_info("Creating virtual environment") create_venv(package_dir, settings, pypath=None, use_global=options.get('global_packages'), verbose=options.get('verbose')) pip_install_venv(package_dir, settings, verbose=options.get('verbose')) else: settings['venv_dir'] = None # save the settings (in the package_dir) save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) echo_success('Package `{package_name}` created in `{package_dir}`'.format(package_name=package_name, package_dir=package_dir))
def config(package, **options): """Manage the configuration (or settings) of the package. The various settings in the .kecpkg-settings.json file are: package_name: name of the package version: version number of the package description: longer description of the package name: name of the author of the package email: email of the author of the package python_version: python version on which the kecpkg will run, corresponds with an executable KE-crunch environment entrypoint_script: the name of the script that will be executed first entrypoint_func: function name inside the script that will be executed. Ensure that it takes *args, **kwargs. venv_dir: python virtual environment directory in the development environment requirements_filename: name of the requirements file with list of package that will be installed before running build_dir: directory where the built kecpkg will be stored exclude_paths: list of paths that will be excluded from the package, next to the build in excludes url: url where the package will be uploaded token: token of the user under which the package is uploaded scope_id: identification of the scope under which the package is uploaded service_id: identification under which the package is re-uploaded (or recently uploaded) last_upload: date and time of the last upload """ echo_info('Locating package ``'.format(package)) package_dir = get_package_dir(package_name=package) package_name = os.path.basename(package_dir) echo_info('Package `{}` has been selected'.format(package_name)) if options.get('init'): if os.path.exists(os.path.join(package_dir, options.get('settings_filename'))) and \ click.confirm('Are you sure you want to overwrite the current settingsfile ' '(old settings will be a backup)?'): copy_path( os.path.join(package_dir, options.get('settings_filename')), os.path.join( package_dir, "{}-backup".format(options.get('settings_filename')))) echo_info('Creating new settingsfile') settings = copy_default_settings() settings['package_name'] = package_name save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) settings = load_settings(package_dir=package_dir) if options.get('interactive'): settings['version'] = click.prompt('Version', default=settings.get( 'version', '0.0.1')) settings['description'] = click.prompt('Description', default=settings.get( 'description', '')) settings['name'] = click.prompt('Author', default=settings.get( 'name', os.environ.get('USER', ''))) settings['email'] = click.prompt('Author\'s email', default=settings.get('email', '')) settings['python_version'] = click.prompt( 'Python version (choose from: {})'.format( settings.get('pyversions')), default='3.5') settings['exclude_paths'] = click.prompt( "Exclude additional paths from kecpkg (eg. 'data, input')", default=settings.get('exclude_paths', ''), value_proc=process_additional_exclude_paths) save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) if options.get('set_key'): k, v = options.get('set_key') if options.get('verbose'): echo_info("Set the key '{}' to value '{}'".format(k, v)) settings[k] = v save_settings(settings, package_dir=package_dir, settings_filename=options.get('settings_filename')) if options.get('get_key'): echo_info( tabulate([ (options.get('get_key'), settings.get(options.get('get_key'))) ], headers=("key", "value"))) return if options.get('verbose'): echo_info(tabulate(settings.items(), headers=("key", "value"))) if not options.get('interactive'): echo_success('Settings file identified and correct')
def upload(package=None, url=None, username=None, password=None, token=None, scope=None, scope_id=None, kecpkg=None, **options): """ Upload built kecpkg to KE-chain. If no options are provided, the interactive mode is triggered. """ package_name = package or get_package_name() or click.prompt( 'Package name') settings = load_settings( package_dir=get_package_dir(package_name), settings_filename=options.get('settings_filename')) if not url or not ((username and password) or token): url = click.prompt('Url (incl http(s)://)', default=settings.get('url') or url) username = click.prompt('Username', default=settings.get('username') or username) password = click.prompt('Password', hide_input=True) # set the interactive world to True for continuation sake options['interactive'] = True elif not options.get('interactive'): url = url or settings.get('url') username = username or settings.get('username') token = token or settings.get('token') scope_id = scope_id or settings.get('scope_id') client = Client(url) client.login(username=username, password=password, token=token) # scope finder if not scope_id and settings.get('scope_id') and \ click.confirm("Do you wish to use the stored `scope_id` in settings: `{}`".format( settings.get('scope_id')), default=True): scope_id = settings.get('scope_id') if not scope_id: scopes = client.scopes() scope_matcher = [ dict(number=i, scope_id=scope.id, scope=scope.name) for i, scope in zip(range(1, len(scopes)), scopes) ] # nice UI echo_info('Choose from following scopes:') for match_dict in scope_matcher: echo_info( "{number} | {scope_id:.8} | {scope}".format(**match_dict)) scope_match = None while not scope_match: scope_guess = click.prompt('Row number, part of Id or Scope') scope_match = validate_scopes(scope_guess, scope_matcher) echo_success( "Scope selected: '{scope}' ({scope_id})".format(**scope_match)) scope_id = scope_match['scope_id'] scope_to_upload = get_project(url, username, password, token, scope_id=scope_id) # service reupload service_id = options.get('service_id') or settings.get('service_id') if options.get('reupload') and not service_id: echo_failure('Please provide a service id to reupload to.') elif service_id and not options.get('reupload') and options.get( 'interactive'): if click.confirm( "Do you wish to *replace* the previously uploaded service: `{}`" .format(service_id), default=True): service_id = service_id else: service_id = None # store to settings if options.get('store'): settings.update( dict(url=url, username=username, scope_id=str(scope_id))) if service_id: settings['service_id'] = str(service_id) save_settings(settings, settings_filename=options.get('settings_filename')) # do upload build_path = os.path.join(get_package_dir(package_name), settings.get('build_dir')) if not os.path.exists(build_path): echo_failure('Cannot find build path, please do `kecpkg build` first') sys.exit(400) upload_package(scope_to_upload, build_path, kecpkg, service_id=service_id, settings=settings, store_settings=options.get('store'), settings_filename=options.get('settings_filename'))
def upload_package(scope, build_path=None, kecpkg_path=None, service_id=None, settings=None, store_settings=True, settings_filename=None): """ Upload the package from build_path to the right scope, create a new KE-chain SIM service. :param scope: Scope object (pykechain) :param build_path: path to the build directory in which the to-be uploaded script resides :param kecpkg_path: path to the kecpkg file to upload (no need to provide build_path) :param service_id: UUID of the service to upload to :param settings: settings of the package :param store_settings: store the settings after update (eg service_id after upload) :param settings_filename: pathname of the file where the settings are stored :return: None """ # if not (kecpkg_path and not build_path) or not (build_path and not kecpkg_path): # echo_failure("You should provide a build path or a kecpkg path") # sys.exit(404) if kecpkg_path and os.path.exists(kecpkg_path): kecpkg_path = kecpkg_path else: built_kecpkgs = os.listdir(build_path) if not kecpkg_path and len(built_kecpkgs) > 1 and settings.get( 'version'): built_kecpkgs = [ f for f in built_kecpkgs if settings.get('version') in f ] if not kecpkg_path and len(built_kecpkgs) == 1: kecpkg_path = os.path.join(build_path, built_kecpkgs[0]) else: echo_info('Provide correct filename to upload') echo_info('\n'.join(os.listdir(build_path))) kecpkg_filename = click.prompt('Filename') kecpkg_path = os.path.join(build_path, kecpkg_filename) if kecpkg_path and os.path.exists(kecpkg_path): # ready to upload echo_info('Ready to upload `{}`'.format(os.path.basename(kecpkg_path))) else: echo_failure('Unable to locate kecpkg to upload') sys.exit(404) # get meta and prepare 2 stage submission # 1. fill service information # 2. do upload if service_id: service = scope.service(pk=service_id) service.upload(kecpkg_path) service.edit(name=settings.get('package_name'), description=settings.get('description', ''), script_version=settings.get('version', '')) else: # Create new service in KE-chain service = scope.create_service( name=settings.get('package_name'), description=settings.get('description', ''), version=settings.get('version', ''), service_type='PYTHON SCRIPT', environment_version=settings.get('python_version'), pkg_path=kecpkg_path) # Wrap up party! echo_success("kecpkg `{}` successfully uploaded to KE-chain.".format( os.path.basename(kecpkg_path))) # noinspection PyProtectedMember success_url = "{api_root}/#scopes/{scope_id}/scripts/{service_id}".format( api_root=scope._client.api_root, scope_id=scope.id, service_id=service.id) echo_success( "To view the newly created service, go to: `{}`".format(success_url)) # update settings if store_settings: settings['service_id'] = str(service.id) from datetime import datetime settings['last_upload'] = str(datetime.now().isoformat()) save_settings(settings, settings_filename=settings_filename)