def _parse(self) -> PatternMatcher: patterns = [] dockerignore = os.path.join(self.root, ".dockerignore") if os.path.isfile(dockerignore): with open(dockerignore, "r") as f: patterns = [ l.strip() for l in f.readlines() if l and not l.startswith("#") ] cli_logger.info( f"No .dockerignore found in {self.root}, not applying any filters") return PatternMatcher(patterns)
def _list_ignored(self) -> Dict: if self.has_git: out = subprocess.run( ["git", "ls-files", "-io", "--exclude-standard"], cwd=self.root, capture_output=True) if out.returncode == 0: return dict.fromkeys( out.stdout.decode("utf-8").split("\n")[:-1]) cli_logger.warning( f"Could not determine ignored files due to:\n{out.stderr}\nNot applying any filters" ) return {} cli_logger.info("No git executable found, not applying any filters") return {}
def get_and_save_remote_with_click_context(ctx: click.Context, project: str, domain: str, save: bool = True) -> FlyteRemote: """ NB: This function will by default mutate the click Context.obj dictionary, adding a remote key with value of the created FlyteRemote object. :param ctx: the click context object :param project: default project for the remote instance :param domain: default domain :param save: If false, will not mutate the context.obj dict :return: FlyteRemote instance """ cfg_file_location = ctx.obj.get(CTX_CONFIG_FILE) cfg_obj = Config.auto(cfg_file_location) cli_logger.info(f"Creating remote with config {cfg_obj}" + ( f" with file {cfg_file_location}" if cfg_file_location else "")) r = FlyteRemote(cfg_obj, default_project=project, default_domain=domain) if save: ctx.obj[FLYTE_REMOTE_INSTANCE_KEY] = r return r
def _refresh_credentials_basic(flyte_client): """ This function is used by the _handle_rpc_error() decorator, depending on the AUTH_MODE config object. This handler is meant for SDK use-cases of auth (like pyflyte, or when users call SDK functions that require access to Admin, like when waiting for another workflow to complete from within a task). This function uses basic auth, which means the credentials for basic auth must be present from wherever this code is running. :param flyte_client: RawSynchronousFlyteClient :return: """ auth_endpoints = _credentials_access.get_authorization_endpoints( flyte_client.url) token_endpoint = auth_endpoints.token_endpoint client_secret = _basic_auth.get_secret() cli_logger.debug( "Basic authorization flow with client id {} scope {}".format( _CLIENT_ID.get(), _get_basic_flow_scopes())) authorization_header = _basic_auth.get_basic_authorization_header( _CLIENT_ID.get(), client_secret) token, expires_in = _basic_auth.get_token(token_endpoint, authorization_header, _get_basic_flow_scopes()) cli_logger.info("Retrieved new token, expires in {}".format(expires_in)) flyte_client.set_access_token(token)
def register( ctx: click.Context, project: str, domain: str, image_config: ImageConfig, output: str, destination_dir: str, service_account: str, raw_data_prefix: str, version: typing.Optional[str], package_or_module: typing.Tuple[str], ): """ see help """ pkgs = ctx.obj[constants.CTX_PACKAGES] if not pkgs: cli_logger.debug("No pkgs") if pkgs: raise ValueError( "Unimplemented, just specify pkgs like folder/files as args at the end of the command" ) if len(package_or_module) == 0: display_help_with_error( ctx, "Missing argument 'PACKAGE_OR_MODULE...', at least one PACKAGE_OR_MODULE is required but multiple can be passed", ) cli_logger.debug( f"Running pyflyte register from {os.getcwd()} " f"with images {image_config} " f"and image destinationfolder {destination_dir} " f"on {len(package_or_module)} package(s) {package_or_module}") # Create and save FlyteRemote, remote = get_and_save_remote_with_click_context(ctx, project, domain) # Todo: add switch for non-fast - skip the zipping and uploading and no fastserializationsettings # Create a zip file containing all the entries. detected_root = find_common_root(package_or_module) cli_logger.debug(f"Using {detected_root} as root folder for project") zip_file = fast_package(detected_root, output) # Upload zip file to Admin using FlyteRemote. md5_bytes, native_url = remote._upload_file(pathlib.Path(zip_file)) cli_logger.debug(f"Uploaded zip {zip_file} to {native_url}") # Create serialization settings # Todo: Rely on default Python interpreter for now, this will break custom Spark containers serialization_settings = SerializationSettings( project=project, domain=domain, image_config=image_config, fast_serialization_settings=FastSerializationSettings( enabled=True, destination_dir=destination_dir, distribution_location=native_url, ), ) options = Options.default_from(k8s_service_account=service_account, raw_data_prefix=raw_data_prefix) # Load all the entities registerable_entities = load_packages_and_modules(serialization_settings, detected_root, list(package_or_module), options) if len(registerable_entities) == 0: display_help_with_error(ctx, "No Flyte entities were detected. Aborting!") cli_logger.info( f"Found and serialized {len(registerable_entities)} entities") if not version: version = remote._version_from_hash(md5_bytes, serialization_settings, service_account, raw_data_prefix) # noqa cli_logger.info(f"Computed version is {version}") # Register using repo code repo_register(registerable_entities, project, domain, version, remote.client)