def test_s3_full_serialize(): s3 = storage.S3( bucket="bucket", key="key", secrets=["hidden", "auth"], labels=["foo", "bar"], add_default_labels=False, )
def test_s3_empty_serialize(): s3 = storage.S3(bucket="bucket") serialized = S3Schema().dump(s3) assert serialized assert serialized["__version__"] == prefect.__version__ assert serialized["bucket"] assert not serialized["key"] assert serialized["secrets"] == []
def test_s3_serialize_with_flows(): s3 = storage.S3(bucket="bucket", key="key", secrets=["hidden", "auth"]) f = prefect.Flow("test") s3.flows["test"] = "key" serialized = S3Schema().dump(s3) assert serialized assert serialized["__version__"] == prefect.__version__ assert serialized["bucket"] == "bucket" assert serialized["key"] == "key" assert serialized["flows"] == {"test": "key"} deserialized = S3Schema().load(serialized) assert f.name in deserialized assert deserialized.secrets == ["hidden", "auth"]
def test_s3_full_serialize(): s3 = storage.S3( bucket="bucket", key="key", secrets=["hidden", "auth"], labels=["foo", "bar"], add_default_labels=False, ) serialized = S3Schema().dump(s3) assert serialized assert serialized["__version__"] == prefect.__version__ assert serialized["bucket"] == "bucket" assert serialized["key"] == "key" assert serialized["secrets"] == ["hidden", "auth"]
def configure_flow_storage(cluster: Cluster, secrets): if cluster.flow_storage_protocol == S3_PROTOCOL: key = secrets[cluster.flow_storage_options.key] secret = secrets[cluster.flow_storage_options.secret] flow_storage = storage.S3( bucket=cluster.flow_storage, client_options={ "aws_access_key_id": key, "aws_secret_access_key": secret }, ) return flow_storage elif cluster.flow_storage_protocol == ABFS_PROTOCOL: secret = secrets[cluster.flow_storage_options.secret] flow_storage = storage.Azure(container=cluster.flow_storage, connection_string=secret) return flow_storage else: raise UnsupportedFlowStorage
def register_recipe(recipe: BaseRecipe): flow_name = "test-noaa-flow" definition = yaml.safe_load( """ networkMode: awsvpc cpu: 1024 memory: 2048 containerDefinitions: - name: flow """ ) outputs = retrieve_stack_outputs() tags = generate_tags(flow_name) fs_remote = S3FileSystem() target = FSSpecTarget( fs_remote, root_path=f"s3://{outputs['cache_bucket_name_output']}/target/oisst/", ) recipe.target = target recipe.input_cache = CacheFSSpecTarget( fs_remote, root_path=(f"s3://{outputs['cache_bucket_name_output']}/cache/oisst/"), ) recipe.metadata_cache = target executor = PrefectPipelineExecutor() pipeline = recipe.to_pipelines() flow = executor.pipelines_to_plan(pipeline) flow.storage = storage.S3(bucket=outputs["storage_bucket_name_output"]) flow.run_config = ECSRun( image=os.environ["BAKERY_IMAGE"], labels=json.loads(os.environ["PREFECT__CLOUD__AGENT__LABELS"]), task_definition=definition, run_task_kwargs={"tags": tags["tag_list"]}, ) flow.executor = DaskExecutor( cluster_class="dask_cloudprovider.aws.FargateCluster", cluster_kwargs={ "image": os.environ["BAKERY_IMAGE"], "vpc": outputs["vpc_output"], "cluster_arn": outputs["cluster_arn_output"], "task_role_arn": outputs["task_role_arn_output"], "execution_role_arn": outputs["task_execution_role_arn_output"], "security_groups": [outputs["security_group_output"]], "scheduler_cpu": 256, "scheduler_mem": 512, "worker_cpu": 1024, "worker_mem": 2048, "scheduler_timeout": "15 minutes", "tags": tags["tag_dict"], }, adapt_kwargs={"maximum": 10}, ) for flow_task in flow.tasks: flow_task.run = set_log_level(flow_task.run) flow.name = flow_name project_name = os.environ["PREFECT_PROJECT"] flow.register(project_name=project_name)
def test_s3_serialize_with_flows(): s3 = storage.S3(bucket="bucket", key="key", secrets=["hidden", "auth"])
def test_s3_empty_serialize(): s3 = storage.S3(bucket="bucket")