def test_local_file_scheme_normalized(self, scheme):
path = "/path/to/file.txt"
unparsed = f"{scheme}://{path}" if scheme else path
parsed = parse_path(unparsed)
assert parsed.scheme == "file"
assert parsed.netloc == ""
assert parsed.path == path
def __init__(
self,
*,
task_definition: dict = None,
task_definition_path: str = None,
task_definition_arn: str = None,
image: str = None,
env: dict = None,
cpu: Union[int, str] = None,
memory: Union[int, str] = None,
task_role_arn: str = None,
execution_role_arn: str = None,
run_task_kwargs: dict = None,
labels: Iterable[str] = None,
) -> None:
super().__init__(labels=labels)
if (
sum(
[
task_definition is not None,
task_definition_path is not None,
task_definition_arn is not None,
]
)
> 1
):
raise ValueError(
"Can only provide one of `task_definition`, `task_definition_path`, "
"or `task_definition_arn`"
)
if task_definition_arn is not None and image is not None:
raise ValueError(
"Cannot provide both `task_definition_arn` and `image`, since an ECS "
"task's `image` can only be configured as part of the task definition"
)
if task_definition_path is not None:
parsed = parse_path(task_definition_path)
if parsed.scheme == "file":
with open(parsed.path) as f:
task_definition = yaml.safe_load(f)
task_definition_path = None
if cpu is not None:
cpu = str(cpu)
if memory is not None:
memory = str(memory)
self.task_definition = task_definition
self.task_definition_path = task_definition_path
self.task_definition_arn = task_definition_arn
self.image = image
self.env = env
self.cpu = cpu
self.memory = memory
self.task_role_arn = task_role_arn
self.execution_role_arn = execution_role_arn
self.run_task_kwargs = run_task_kwargs
def register(ctx, project, paths, modules, json_paths, names, labels, force,
watch, schedule):
"""Register one or more flows into a project.
Flows with unchanged metadata will be skipped as registering again will only
change the version number.
"""
# Since the old command was a subcommand of this, we have to do some
# mucking to smoothly deprecate it. Can be removed with `prefect register
# flow` is removed.
if ctx.invoked_subcommand is not None:
if any([project, paths, modules, names, labels, force]):
raise ClickException("Got unexpected extra argument (%s)" %
ctx.invoked_subcommand)
return
if project is None:
raise ClickException("Missing required option '--project'")
paths = expand_paths(paths)
if watch:
if any(parse_path(j).scheme != "file" for j in json_paths):
raise ClickException("--watch is not supported for remote paths")
json_paths = set(json_paths)
ctx = multiprocessing.get_context("spawn")
for change_paths_temp, change_mods in watch_for_changes(
paths=paths, modules=modules):
change_paths = []
change_json_paths = []
for p in change_paths_temp:
if p in json_paths:
change_json_paths.append(p)
else:
change_paths.append(p)
proc = ctx.Process(
target=register_internal,
name="prefect-register",
args=(project, ),
kwargs=dict(
paths=change_paths,
modules=change_mods,
json_paths=change_json_paths,
names=names,
labels=labels,
force=force,
in_watch=True,
schedule=schedule,
),
daemon=True,
)
proc.start()
proc.join()
else:
modules = list(modules or ())
register_internal(project, paths, modules, json_paths, names, labels,
force, schedule)
def __init__(
self,
*,
job_template_path: str = None,
job_template: Union[str, dict] = None,
image: str = None,
env: dict = None,
cpu_limit: Union[float, str] = None,
cpu_request: Union[float, str] = None,
memory_limit: str = None,
memory_request: str = None,
service_account_name: str = None,
image_pull_secrets: Iterable[str] = None,
labels: Iterable[str] = None,
image_pull_policy: str = None,
) -> None:
super().__init__(env=env, labels=labels)
if job_template_path is not None and job_template is not None:
raise ValueError(
"Cannot provide both `job_template_path` and `job_template`")
if job_template_path is not None:
parsed = parse_path(job_template_path)
if parsed.scheme == "file":
with open(parsed.path) as f:
job_template = yaml.safe_load(f)
job_template_path = None
elif job_template is not None:
# Normalize job templates to objects rather than str
if isinstance(job_template, str):
job_template = yaml.safe_load(job_template)
assert job_template is None or isinstance(job_template, dict) # mypy
if cpu_limit is not None:
cpu_limit = str(cpu_limit)
if cpu_request is not None:
cpu_request = str(cpu_request)
if image_pull_secrets is not None:
image_pull_secrets = list(image_pull_secrets)
image_pull_policies = {"Always", "IfNotPresent", "Never"}
if (image_pull_policy is not None
and image_pull_policy not in image_pull_policies):
raise ValueError(
f"Invalid image_pull_policy {image_pull_policy!r}. "
"Expected 'Always', 'IfNotPresent', or 'Never'")
self.job_template_path = job_template_path
self.job_template = job_template
self.image = image
self.cpu_limit = cpu_limit
self.cpu_request = cpu_request
self.memory_limit = memory_limit
self.memory_request = memory_request
self.service_account_name = service_account_name
self.image_pull_secrets = image_pull_secrets
self.image_pull_policy = image_pull_policy
def register(project, paths, modules, json_paths, names, labels, force, watch,
schedule):
"""Register one or more flows into a project.
Flows with unchanged metadata will be skipped as registering again will only
change the version number.
"""
if project is None:
raise ClickException("Missing required option '--project'")
paths = expand_paths(paths)
if watch:
if any(parse_path(j).scheme != "file" for j in json_paths):
raise ClickException("--watch is not supported for remote paths")
json_paths = set(json_paths)
ctx = multiprocessing.get_context("spawn")
for change_paths_temp, change_mods in watch_for_changes(
paths=paths, modules=modules):
change_paths = []
change_json_paths = []
for p in change_paths_temp:
if p in json_paths:
change_json_paths.append(p)
else:
change_paths.append(p)
proc = ctx.Process(
target=register_internal,
name="prefect-register",
args=(project, ),
kwargs=dict(
paths=change_paths,
modules=change_mods,
json_paths=change_json_paths,
names=names,
labels=labels,
force=force,
in_watch=True,
schedule=schedule,
),
daemon=True,
)
proc.start()
proc.join()
else:
modules = list(modules or ())
register_internal(project, paths, modules, json_paths, names, labels,
force, schedule)
def __init__(
self,
*,
job_template_path: str = None,
job_template: Union[str, dict] = None,
image: str = None,
env: dict = None,
cpu_limit: Union[float, str] = None,
cpu_request: Union[float, str] = None,
memory_limit: str = None,
memory_request: str = None,
service_account_name: str = None,
image_pull_secrets: Iterable[str] = None,
labels: Iterable[str] = None,
) -> None:
super().__init__(labels=labels)
if job_template_path is not None and job_template is not None:
raise ValueError(
"Cannot provide both `job_template_path` and `job_template`")
if job_template_path is not None:
parsed = parse_path(job_template_path)
if parsed.scheme == "file":
with open(parsed.path) as f:
job_template = yaml.safe_load(f)
job_template_path = None
elif job_template is not None:
# Normalize job templates to objects rather than str
if isinstance(job_template, str):
job_template = yaml.safe_load(job_template)
assert job_template is None or isinstance(job_template, dict) # mypy
if cpu_limit is not None:
cpu_limit = str(cpu_limit)
if cpu_request is not None:
cpu_request = str(cpu_request)
if image_pull_secrets is not None:
image_pull_secrets = list(image_pull_secrets)
self.job_template_path = job_template_path
self.job_template = job_template
self.image = image
self.env = env
self.cpu_limit = cpu_limit
self.cpu_request = cpu_request
self.memory_limit = memory_limit
self.memory_request = memory_request
self.service_account_name = service_account_name
self.image_pull_secrets = image_pull_secrets
def __init__(
self,
*,
task_definition: dict = None,
task_definition_path: str = None,
image: str = None,
env: dict = None,
cpu: Union[int, str] = None,
memory: Union[int, str] = None,
task_role_arn: str = None,
run_task_kwargs: dict = None,
labels: Iterable[str] = None,
) -> None:
super().__init__(labels=labels)
if task_definition is not None and task_definition_path is not None:
raise ValueError(
"Cannot provide both `task_definition` and `task_definition_path`"
)
if task_definition_path is not None:
parsed = parse_path(task_definition_path)
if parsed.scheme == "file":
with open(parsed.path) as f:
task_definition = yaml.safe_load(f)
task_definition_path = None
if cpu is not None:
cpu = str(cpu)
if memory is not None:
memory = str(memory)
self.task_definition = task_definition
self.task_definition_path = task_definition_path
self.image = image
self.env = env
self.cpu = cpu
self.memory = memory
self.task_role_arn = task_role_arn
self.run_task_kwargs = run_task_kwargs
def test_all_components(self):
parsed = parse_path("s3://bucket/path/to/file.txt")
assert parsed.scheme == "s3"
assert parsed.netloc == "bucket"
assert parsed.path == "/path/to/file.txt"
def test_windows_local_paths(self, path):
parsed = parse_path(path)
assert parsed.scheme == "file"
assert parsed.netloc == ""
assert parsed.path == path
