def _parse(self) -> PatternMatcher:
patterns = []
dockerignore = os.path.join(self.root, ".dockerignore")
if os.path.isfile(dockerignore):
with open(dockerignore, "r") as f:
patterns = [
l.strip() for l in f.readlines()
if l and not l.startswith("#")
]
cli_logger.info(
f"No .dockerignore found in {self.root}, not applying any filters")
return PatternMatcher(patterns)
def _list_ignored(self) -> Dict:
if self.has_git:
out = subprocess.run(
["git", "ls-files", "-io", "--exclude-standard"],
cwd=self.root,
capture_output=True)
if out.returncode == 0:
return dict.fromkeys(
out.stdout.decode("utf-8").split("\n")[:-1])
cli_logger.warning(
f"Could not determine ignored files due to:\n{out.stderr}\nNot applying any filters"
)
return {}
cli_logger.info("No git executable found, not applying any filters")
return {}
def get_and_save_remote_with_click_context(ctx: click.Context,
project: str,
domain: str,
save: bool = True) -> FlyteRemote:
"""
NB: This function will by default mutate the click Context.obj dictionary, adding a remote key with value
of the created FlyteRemote object.
:param ctx: the click context object
:param project: default project for the remote instance
:param domain: default domain
:param save: If false, will not mutate the context.obj dict
:return: FlyteRemote instance
"""
cfg_file_location = ctx.obj.get(CTX_CONFIG_FILE)
cfg_obj = Config.auto(cfg_file_location)
cli_logger.info(f"Creating remote with config {cfg_obj}" + (
f" with file {cfg_file_location}" if cfg_file_location else ""))
r = FlyteRemote(cfg_obj, default_project=project, default_domain=domain)
if save:
ctx.obj[FLYTE_REMOTE_INSTANCE_KEY] = r
return r
def _refresh_credentials_basic(flyte_client):
"""
This function is used by the _handle_rpc_error() decorator, depending on the AUTH_MODE config object. This handler
is meant for SDK use-cases of auth (like pyflyte, or when users call SDK functions that require access to Admin,
like when waiting for another workflow to complete from within a task). This function uses basic auth, which means
the credentials for basic auth must be present from wherever this code is running.
:param flyte_client: RawSynchronousFlyteClient
:return:
"""
auth_endpoints = _credentials_access.get_authorization_endpoints(
flyte_client.url)
token_endpoint = auth_endpoints.token_endpoint
client_secret = _basic_auth.get_secret()
cli_logger.debug(
"Basic authorization flow with client id {} scope {}".format(
_CLIENT_ID.get(), _get_basic_flow_scopes()))
authorization_header = _basic_auth.get_basic_authorization_header(
_CLIENT_ID.get(), client_secret)
token, expires_in = _basic_auth.get_token(token_endpoint,
authorization_header,
_get_basic_flow_scopes())
cli_logger.info("Retrieved new token, expires in {}".format(expires_in))
flyte_client.set_access_token(token)
def register(
ctx: click.Context,
project: str,
domain: str,
image_config: ImageConfig,
output: str,
destination_dir: str,
service_account: str,
raw_data_prefix: str,
version: typing.Optional[str],
package_or_module: typing.Tuple[str],
):
"""
see help
"""
pkgs = ctx.obj[constants.CTX_PACKAGES]
if not pkgs:
cli_logger.debug("No pkgs")
if pkgs:
raise ValueError(
"Unimplemented, just specify pkgs like folder/files as args at the end of the command"
)
if len(package_or_module) == 0:
display_help_with_error(
ctx,
"Missing argument 'PACKAGE_OR_MODULE...', at least one PACKAGE_OR_MODULE is required but multiple can be passed",
)
cli_logger.debug(
f"Running pyflyte register from {os.getcwd()} "
f"with images {image_config} "
f"and image destinationfolder {destination_dir} "
f"on {len(package_or_module)} package(s) {package_or_module}")
# Create and save FlyteRemote,
remote = get_and_save_remote_with_click_context(ctx, project, domain)
# Todo: add switch for non-fast - skip the zipping and uploading and no fastserializationsettings
# Create a zip file containing all the entries.
detected_root = find_common_root(package_or_module)
cli_logger.debug(f"Using {detected_root} as root folder for project")
zip_file = fast_package(detected_root, output)
# Upload zip file to Admin using FlyteRemote.
md5_bytes, native_url = remote._upload_file(pathlib.Path(zip_file))
cli_logger.debug(f"Uploaded zip {zip_file} to {native_url}")
# Create serialization settings
# Todo: Rely on default Python interpreter for now, this will break custom Spark containers
serialization_settings = SerializationSettings(
project=project,
domain=domain,
image_config=image_config,
fast_serialization_settings=FastSerializationSettings(
enabled=True,
destination_dir=destination_dir,
distribution_location=native_url,
),
)
options = Options.default_from(k8s_service_account=service_account,
raw_data_prefix=raw_data_prefix)
# Load all the entities
registerable_entities = load_packages_and_modules(serialization_settings,
detected_root,
list(package_or_module),
options)
if len(registerable_entities) == 0:
display_help_with_error(ctx,
"No Flyte entities were detected. Aborting!")
cli_logger.info(
f"Found and serialized {len(registerable_entities)} entities")
if not version:
version = remote._version_from_hash(md5_bytes, serialization_settings,
service_account,
raw_data_prefix) # noqa
cli_logger.info(f"Computed version is {version}")
# Register using repo code
repo_register(registerable_entities, project, domain, version,
remote.client)