def test_basic_conversion_s3_result(): result_handler = S3ResultHandler(bucket="foo", boto3_kwargs=dict(x=42, y=[1, 2, 3])) result = ResultHandlerResult.from_result_handler(result_handler) assert isinstance(result, S3Result) assert result.bucket == "foo" assert result.boto3_kwargs == dict(x=42, y=[1, 2, 3])
def test_converted_result_writes_and_reads(self, handler): result = ResultHandlerResult.from_result_handler(handler) new = result.write(42) assert new is not result assert new.value == 42 assert new.location assert new.result_handler.stuff[new.location] == 42 another = new.read(new.location) assert another is not new assert another.value == 42 assert another.location == new.location
def __init__( self, name: str = None, slug: str = None, tags: Iterable[str] = None, max_retries: int = None, retry_delay: timedelta = None, timeout: int = None, trigger: "Callable[[Dict[Edge, State]], bool]" = None, skip_on_upstream_skip: bool = True, cache_for: timedelta = None, cache_validator: Callable = None, cache_key: str = None, checkpoint: bool = None, result_handler: "ResultHandler" = None, state_handlers: List[Callable] = None, on_failure: Callable = None, log_stdout: bool = False, result: "Result" = None, target: str = None, ): self.name = name or type(self).__name__ self.slug = slug self.logger = logging.get_logger(self.name) # avoid silently iterating over a string if isinstance(tags, str): raise TypeError("Tags should be a set of tags, not a string.") current_tags = set(prefect.context.get("tags", set())) self.tags = (set(tags) if tags is not None else set()) | current_tags max_retries = (max_retries if max_retries is not None else prefect.config.tasks.defaults.max_retries) retry_delay = (retry_delay if retry_delay is not None else prefect.config.tasks.defaults.retry_delay) timeout = (timeout if timeout is not None else prefect.config.tasks.defaults.timeout) if max_retries > 0 and retry_delay is None: raise ValueError( "A datetime.timedelta `retry_delay` must be provided if max_retries > 0" ) # specify not max retries because the default is false if retry_delay is not None and not max_retries: raise ValueError( "A `max_retries` argument greater than 0 must be provided if specifying " "a retry delay.") if timeout is not None and not isinstance(timeout, int): raise TypeError( "Only integer timeouts (representing seconds) are supported.") self.max_retries = max_retries self.retry_delay = retry_delay self.timeout = timeout self.trigger = trigger or prefect.triggers.all_successful self.skip_on_upstream_skip = skip_on_upstream_skip if cache_for is None and ( cache_validator is not None and cache_validator is not prefect.engine.cache_validators.never_use): warnings.warn( "cache_validator provided without specifying cache expiration " "(cache_for); this Task will not be cached.") self.cache_for = cache_for self.cache_key = cache_key default_validator = (prefect.engine.cache_validators.never_use if cache_for is None else prefect.engine.cache_validators.duration_only) self.cache_validator = cache_validator or default_validator self.checkpoint = checkpoint if result_handler: warnings.warn( "Result Handlers are deprecated; please use the new style Result classes instead." ) self.result = ResultHandlerResult.from_result_handler( result_handler) # type: Optional[Result] else: self.result = result self.target = target # if both a target and a result were provided, update the result location # to point at the target if self.target and self.result: if (getattr(self.result, "location", None) and self.result.location != self.target): warnings.warn( "Both `result.location` and `target` were provided. " "The `target` value will be used.") self.result = self.result.copy() self.result.location = self.target if state_handlers and not isinstance(state_handlers, collections.abc.Sequence): raise TypeError("state_handlers should be iterable.") self.state_handlers = state_handlers or [] if on_failure is not None: self.state_handlers.append( callback_factory(on_failure, check=lambda s: s.is_failed())) self.auto_generated = False self.log_stdout = log_stdout # if new task creations are being tracked, add this task # this makes it possible to give guidance to users that forget # to add tasks to a flow if "_unused_task_tracker" in prefect.context: if not isinstance(self, prefect.tasks.core.constants.Constant): prefect.context._unused_task_tracker.add(self)
def test_conversion_type(self, handler): result = ResultHandlerResult.from_result_handler(handler) assert isinstance(result, ResultHandlerResult) assert isinstance(result.result_handler, type(handler))
def test_basic_conversion_gcs_result(): result_handler = GCSResultHandler(bucket="foo") result = ResultHandlerResult.from_result_handler(result_handler) assert isinstance(result, GCSResult) assert result.bucket == "foo"
def test_basic_conversion_local_result(tmpdir): result_handler = LocalResultHandler(dir=str(tmpdir)) result = ResultHandlerResult.from_result_handler(result_handler) assert isinstance(result, LocalResult) assert result.dir == str(tmpdir)
def test_basic_conversion_constant_result(): result_handler = ConstantResultHandler(value=42) result = ResultHandlerResult.from_result_handler(result_handler) assert isinstance(result, ConstantResult) assert result.value == 42
def test_basic_conversion_json_result(): result_handler = JSONResultHandler() result = ResultHandlerResult.from_result_handler(result_handler) assert isinstance(result, PrefectResult) assert result.write(42).location == "42"
def test_basic_conversion_secret_result(): task = PrefectSecret("foo") result_handler = SecretResultHandler(task) result = ResultHandlerResult.from_result_handler(result_handler) assert isinstance(result, SecretResult) assert result.secret_task is task