def validate_package(ctx, param, values):
pkgs = []
for val in values:
if "/" in val or "-" in val or "\\" in val:
raise click.BadParameter(
f"Illegal package value {val} for parameter: {param}. Expected for the form [a.b.c]"
)
elif "," in val:
pkgs.extend(val.split(","))
else:
pkgs.append(val)
cli_logger.debug(f"Using packages: {pkgs}")
return pkgs
def _refresh_credentials_from_command(flyte_client):
"""
This function is used when the configuration value for AUTH_MODE is set to 'external_process'.
It reads an id token generated by an external process started by running the 'command'.
:param flyte_client: RawSynchronousFlyteClient
:return:
"""
command = _COMMAND.get()
cli_logger.debug(
"Starting external process to generate id token. Command {}".format(
command))
try:
output = subprocess.run(command,
capture_output=True,
text=True,
check=True)
except subprocess.CalledProcessError as e:
cli_logger.error(
"Failed to generate token from command {}".format(command))
raise _user_exceptions.FlyteAuthenticationException(
"Problems refreshing token with command: " + _six.text_type(e))
flyte_client.set_access_token(output.stdout.strip())
def _refresh_credentials_basic(flyte_client):
"""
This function is used by the _handle_rpc_error() decorator, depending on the AUTH_MODE config object. This handler
is meant for SDK use-cases of auth (like pyflyte, or when users call SDK functions that require access to Admin,
like when waiting for another workflow to complete from within a task). This function uses basic auth, which means
the credentials for basic auth must be present from wherever this code is running.
:param flyte_client: RawSynchronousFlyteClient
:return:
"""
auth_endpoints = _credentials_access.get_authorization_endpoints(
flyte_client.url)
token_endpoint = auth_endpoints.token_endpoint
client_secret = _basic_auth.get_secret()
cli_logger.debug(
"Basic authorization flow with client id {} scope {}".format(
_CLIENT_ID.get(), _get_basic_flow_scopes()))
authorization_header = _basic_auth.get_basic_authorization_header(
_CLIENT_ID.get(), client_secret)
token, expires_in = _basic_auth.get_token(token_endpoint,
authorization_header,
_get_basic_flow_scopes())
cli_logger.info("Retrieved new token, expires in {}".format(expires_in))
flyte_client.set_access_token(token)
def register(
ctx: click.Context,
project: str,
domain: str,
image_config: ImageConfig,
output: str,
destination_dir: str,
service_account: str,
raw_data_prefix: str,
version: typing.Optional[str],
package_or_module: typing.Tuple[str],
):
"""
see help
"""
pkgs = ctx.obj[constants.CTX_PACKAGES]
if not pkgs:
cli_logger.debug("No pkgs")
if pkgs:
raise ValueError(
"Unimplemented, just specify pkgs like folder/files as args at the end of the command"
)
if len(package_or_module) == 0:
display_help_with_error(
ctx,
"Missing argument 'PACKAGE_OR_MODULE...', at least one PACKAGE_OR_MODULE is required but multiple can be passed",
)
cli_logger.debug(
f"Running pyflyte register from {os.getcwd()} "
f"with images {image_config} "
f"and image destinationfolder {destination_dir} "
f"on {len(package_or_module)} package(s) {package_or_module}")
# Create and save FlyteRemote,
remote = get_and_save_remote_with_click_context(ctx, project, domain)
# Todo: add switch for non-fast - skip the zipping and uploading and no fastserializationsettings
# Create a zip file containing all the entries.
detected_root = find_common_root(package_or_module)
cli_logger.debug(f"Using {detected_root} as root folder for project")
zip_file = fast_package(detected_root, output)
# Upload zip file to Admin using FlyteRemote.
md5_bytes, native_url = remote._upload_file(pathlib.Path(zip_file))
cli_logger.debug(f"Uploaded zip {zip_file} to {native_url}")
# Create serialization settings
# Todo: Rely on default Python interpreter for now, this will break custom Spark containers
serialization_settings = SerializationSettings(
project=project,
domain=domain,
image_config=image_config,
fast_serialization_settings=FastSerializationSettings(
enabled=True,
destination_dir=destination_dir,
distribution_location=native_url,
),
)
options = Options.default_from(k8s_service_account=service_account,
raw_data_prefix=raw_data_prefix)
# Load all the entities
registerable_entities = load_packages_and_modules(serialization_settings,
detected_root,
list(package_or_module),
options)
if len(registerable_entities) == 0:
display_help_with_error(ctx,
"No Flyte entities were detected. Aborting!")
cli_logger.info(
f"Found and serialized {len(registerable_entities)} entities")
if not version:
version = remote._version_from_hash(md5_bytes, serialization_settings,
service_account,
raw_data_prefix) # noqa
cli_logger.info(f"Computed version is {version}")
# Register using repo code
repo_register(registerable_entities, project, domain, version,
remote.client)