def test_local_file_scheme_normalized(self, scheme): path = "/path/to/file.txt" unparsed = f"{scheme}://{path}" if scheme else path parsed = parse_path(unparsed) assert parsed.scheme == "file" assert parsed.netloc == "" assert parsed.path == path
def __init__( self, *, task_definition: dict = None, task_definition_path: str = None, task_definition_arn: str = None, image: str = None, env: dict = None, cpu: Union[int, str] = None, memory: Union[int, str] = None, task_role_arn: str = None, execution_role_arn: str = None, run_task_kwargs: dict = None, labels: Iterable[str] = None, ) -> None: super().__init__(labels=labels) if ( sum( [ task_definition is not None, task_definition_path is not None, task_definition_arn is not None, ] ) > 1 ): raise ValueError( "Can only provide one of `task_definition`, `task_definition_path`, " "or `task_definition_arn`" ) if task_definition_arn is not None and image is not None: raise ValueError( "Cannot provide both `task_definition_arn` and `image`, since an ECS " "task's `image` can only be configured as part of the task definition" ) if task_definition_path is not None: parsed = parse_path(task_definition_path) if parsed.scheme == "file": with open(parsed.path) as f: task_definition = yaml.safe_load(f) task_definition_path = None if cpu is not None: cpu = str(cpu) if memory is not None: memory = str(memory) self.task_definition = task_definition self.task_definition_path = task_definition_path self.task_definition_arn = task_definition_arn self.image = image self.env = env self.cpu = cpu self.memory = memory self.task_role_arn = task_role_arn self.execution_role_arn = execution_role_arn self.run_task_kwargs = run_task_kwargs
def register(ctx, project, paths, modules, json_paths, names, labels, force, watch, schedule): """Register one or more flows into a project. Flows with unchanged metadata will be skipped as registering again will only change the version number. """ # Since the old command was a subcommand of this, we have to do some # mucking to smoothly deprecate it. Can be removed with `prefect register # flow` is removed. if ctx.invoked_subcommand is not None: if any([project, paths, modules, names, labels, force]): raise ClickException("Got unexpected extra argument (%s)" % ctx.invoked_subcommand) return if project is None: raise ClickException("Missing required option '--project'") paths = expand_paths(paths) if watch: if any(parse_path(j).scheme != "file" for j in json_paths): raise ClickException("--watch is not supported for remote paths") json_paths = set(json_paths) ctx = multiprocessing.get_context("spawn") for change_paths_temp, change_mods in watch_for_changes( paths=paths, modules=modules): change_paths = [] change_json_paths = [] for p in change_paths_temp: if p in json_paths: change_json_paths.append(p) else: change_paths.append(p) proc = ctx.Process( target=register_internal, name="prefect-register", args=(project, ), kwargs=dict( paths=change_paths, modules=change_mods, json_paths=change_json_paths, names=names, labels=labels, force=force, in_watch=True, schedule=schedule, ), daemon=True, ) proc.start() proc.join() else: modules = list(modules or ()) register_internal(project, paths, modules, json_paths, names, labels, force, schedule)
def __init__( self, *, job_template_path: str = None, job_template: Union[str, dict] = None, image: str = None, env: dict = None, cpu_limit: Union[float, str] = None, cpu_request: Union[float, str] = None, memory_limit: str = None, memory_request: str = None, service_account_name: str = None, image_pull_secrets: Iterable[str] = None, labels: Iterable[str] = None, image_pull_policy: str = None, ) -> None: super().__init__(env=env, labels=labels) if job_template_path is not None and job_template is not None: raise ValueError( "Cannot provide both `job_template_path` and `job_template`") if job_template_path is not None: parsed = parse_path(job_template_path) if parsed.scheme == "file": with open(parsed.path) as f: job_template = yaml.safe_load(f) job_template_path = None elif job_template is not None: # Normalize job templates to objects rather than str if isinstance(job_template, str): job_template = yaml.safe_load(job_template) assert job_template is None or isinstance(job_template, dict) # mypy if cpu_limit is not None: cpu_limit = str(cpu_limit) if cpu_request is not None: cpu_request = str(cpu_request) if image_pull_secrets is not None: image_pull_secrets = list(image_pull_secrets) image_pull_policies = {"Always", "IfNotPresent", "Never"} if (image_pull_policy is not None and image_pull_policy not in image_pull_policies): raise ValueError( f"Invalid image_pull_policy {image_pull_policy!r}. " "Expected 'Always', 'IfNotPresent', or 'Never'") self.job_template_path = job_template_path self.job_template = job_template self.image = image self.cpu_limit = cpu_limit self.cpu_request = cpu_request self.memory_limit = memory_limit self.memory_request = memory_request self.service_account_name = service_account_name self.image_pull_secrets = image_pull_secrets self.image_pull_policy = image_pull_policy
def register(project, paths, modules, json_paths, names, labels, force, watch, schedule): """Register one or more flows into a project. Flows with unchanged metadata will be skipped as registering again will only change the version number. """ if project is None: raise ClickException("Missing required option '--project'") paths = expand_paths(paths) if watch: if any(parse_path(j).scheme != "file" for j in json_paths): raise ClickException("--watch is not supported for remote paths") json_paths = set(json_paths) ctx = multiprocessing.get_context("spawn") for change_paths_temp, change_mods in watch_for_changes( paths=paths, modules=modules): change_paths = [] change_json_paths = [] for p in change_paths_temp: if p in json_paths: change_json_paths.append(p) else: change_paths.append(p) proc = ctx.Process( target=register_internal, name="prefect-register", args=(project, ), kwargs=dict( paths=change_paths, modules=change_mods, json_paths=change_json_paths, names=names, labels=labels, force=force, in_watch=True, schedule=schedule, ), daemon=True, ) proc.start() proc.join() else: modules = list(modules or ()) register_internal(project, paths, modules, json_paths, names, labels, force, schedule)
def __init__( self, *, job_template_path: str = None, job_template: Union[str, dict] = None, image: str = None, env: dict = None, cpu_limit: Union[float, str] = None, cpu_request: Union[float, str] = None, memory_limit: str = None, memory_request: str = None, service_account_name: str = None, image_pull_secrets: Iterable[str] = None, labels: Iterable[str] = None, ) -> None: super().__init__(labels=labels) if job_template_path is not None and job_template is not None: raise ValueError( "Cannot provide both `job_template_path` and `job_template`") if job_template_path is not None: parsed = parse_path(job_template_path) if parsed.scheme == "file": with open(parsed.path) as f: job_template = yaml.safe_load(f) job_template_path = None elif job_template is not None: # Normalize job templates to objects rather than str if isinstance(job_template, str): job_template = yaml.safe_load(job_template) assert job_template is None or isinstance(job_template, dict) # mypy if cpu_limit is not None: cpu_limit = str(cpu_limit) if cpu_request is not None: cpu_request = str(cpu_request) if image_pull_secrets is not None: image_pull_secrets = list(image_pull_secrets) self.job_template_path = job_template_path self.job_template = job_template self.image = image self.env = env self.cpu_limit = cpu_limit self.cpu_request = cpu_request self.memory_limit = memory_limit self.memory_request = memory_request self.service_account_name = service_account_name self.image_pull_secrets = image_pull_secrets
def __init__( self, *, task_definition: dict = None, task_definition_path: str = None, image: str = None, env: dict = None, cpu: Union[int, str] = None, memory: Union[int, str] = None, task_role_arn: str = None, run_task_kwargs: dict = None, labels: Iterable[str] = None, ) -> None: super().__init__(labels=labels) if task_definition is not None and task_definition_path is not None: raise ValueError( "Cannot provide both `task_definition` and `task_definition_path`" ) if task_definition_path is not None: parsed = parse_path(task_definition_path) if parsed.scheme == "file": with open(parsed.path) as f: task_definition = yaml.safe_load(f) task_definition_path = None if cpu is not None: cpu = str(cpu) if memory is not None: memory = str(memory) self.task_definition = task_definition self.task_definition_path = task_definition_path self.image = image self.env = env self.cpu = cpu self.memory = memory self.task_role_arn = task_role_arn self.run_task_kwargs = run_task_kwargs
def test_all_components(self): parsed = parse_path("s3://bucket/path/to/file.txt") assert parsed.scheme == "s3" assert parsed.netloc == "bucket" assert parsed.path == "/path/to/file.txt"
def test_windows_local_paths(self, path): parsed = parse_path(path) assert parsed.scheme == "file" assert parsed.netloc == "" assert parsed.path == path