def test_azure_exists(self, monkeypatch): client = MagicMock(get_blob_properties=MagicMock()) service = MagicMock(get_blob_client=MagicMock(return_value=client)) monkeypatch.setattr( "prefect.engine.results.azure_result.AzureResult.service", service) result = AzureResult(container="foo", location="{thing}/here.txt") assert result.exists("44.txt") is True
def test_azure_init(self, azure_client): result = AzureResult(container="bob", connection_string="conn") assert result.value == None assert result.connection_string == "conn" assert result.connection_string_secret == None assert azure_client.called is False result.initialize_service() assert azure_client.called is True
def test_azure_writes_binary_string(self, monkeypatch): client = MagicMock(upload_blob=MagicMock()) service = MagicMock(get_blob_client=MagicMock(return_value=client)) monkeypatch.setattr( "prefect.engine.results.azure_result.AzureResult.service", service) result = AzureResult(container="foo", location="nothing/here.txt") new_result = result.write(None) assert client.upload_blob.called assert isinstance(client.upload_blob.call_args[0][0], str)
def test_azure_does_not_exists(self, monkeypatch): from azure.core.exceptions import ResourceNotFoundError client = MagicMock(get_blob_properties=MagicMock( side_effect=ResourceNotFoundError)) service = MagicMock(get_blob_client=MagicMock(return_value=client)) monkeypatch.setattr( "prefect.engine.results.azure_result.AzureResult.service", service) result = AzureResult(container="foo", location="{thing}/here.txt") assert result.exists("44.txt") is False
def test_azure_reads_and_updates_location(self, monkeypatch): client = MagicMock(download_blob=MagicMock(return_value="")) service = MagicMock(get_blob_client=MagicMock(return_value=client)) monkeypatch.setattr( "prefect.engine.results.azure_result.AzureResult.service", service) result = AzureResult(container="foo", location="{thing}/here.txt") new_result = result.read("path/to/my/stuff.txt") assert new_result.location == "path/to/my/stuff.txt" assert new_result.value is None
def test_azure_writes_to_blob_using_rendered_template_name(self, monkeypatch): client = MagicMock(upload_blob=MagicMock()) service = MagicMock(get_blob_client=MagicMock(return_value=client)) monkeypatch.setattr( "prefect.engine.results.azure_result.AzureResult.service", service ) result = AzureResult(container="foo", location="{thing}/here.txt") new_result = result.write("so-much-data", thing=42) assert new_result.location == "42/here.txt" assert client.upload_blob.called assert service.get_blob_client.call_args[1]["blob"] == "42/here.txt" assert service.get_blob_client.call_args[1]["container"] == "foo"
def test_azure_init_connection_string(self, azure_client): result = AzureResult(container="bob", connection_string="con1") result.initialize_service() azure_client.assert_called_with(conn_str="con1") with prefect.context({"secrets": {"test": "con2"}}): result = AzureResult(container="bob", connection_string_secret="test") result.initialize_service() azure_client.assert_called_with(conn_str="con2")
def __init__(self, container: str, connection_string: str = None, blob_name: str = None, stored_as_script: bool = False, **kwargs: Any) -> None: self.connection_string = connection_string or os.getenv( "AZURE_STORAGE_CONNECTION_STRING") self.container = container self.blob_name = blob_name result = AzureResult(connection_string=self.connection_string, container=container) super().__init__(result=result, stored_as_script=stored_as_script, **kwargs)
def __init__(self, container: str, connection_string: str = None, blob_name: str = None, **kwargs: Any) -> None: self.flows = dict() # type: Dict[str, str] self._flows = dict() # type: Dict[str, "Flow"] self.connection_string = connection_string or os.getenv( "AZURE_STORAGE_CONNECTION_STRING") self.container = container self.blob_name = blob_name result = AzureResult(connection_string=self.connection_string, container=container) super().__init__(result=result, **kwargs)
def __init__( self, container: str, connection_string_secret: str = None, blob_name: str = None, overwrite: bool = False, stored_as_script: bool = False, **kwargs: Any ) -> None: self.container = container self.connection_string_secret = connection_string_secret self.blob_name = blob_name self.overwrite = overwrite result = AzureResult( connection_string_secret=self.connection_string_secret, container=container, ) super().__init__(result=result, stored_as_script=stored_as_script, **kwargs)
def test_azure_result_is_pickleable(self, azure_client): result = AzureResult("foo") res = cloudpickle.loads(cloudpickle.dumps(result)) assert isinstance(res, AzureResult)
def test_azure_init_with_values(self, azure_client): result = AzureResult(container="bob", connection_string="conn", value=3) assert result.value == 3
from prefect.engine.results import AzureResult import os # con_string = "" azure_result = AzureResult(container="flows", connection_string=os.environ["AZ"], location="test5") # w = azure_result.write({"Test": "123"}) r = azure_result.read("test5") print(r)
def test_azure_init_without_connection_string(self, azure_client): result = AzureResult(container="bob", connection_string=None) with pytest.raises(ValueError): result.initialize_service()
# Storage FLOWS_DIR_PATH = '/opt/server/src/flows' storage_kwargs = { 'dockerfile': 'server/Dockerfile', 'registry_url': REGISTRY_URL, 'stored_as_script': True, } # Executer local_executor = LocalExecutor() dask_executor = DaskExecutor(address=DASK_SCHEDULER_ADDR) # Result if RESULT_SUBCLASS == 'azure': result = AzureResult(container=AZURE_RESULT_CONTAINER) elif RESULT_SUBCLASS == 's3': result = S3Result(bucket=S3_RESULT_BUCKET) else: result = LocalResult(dir=LOCAL_RESULT_DIR) # Set flow run configs mapreduce_wordcount.run_config = run_config # Set flow storage mapreduce_wordcount.storage = Docker( path=f'{FLOWS_DIR_PATH}/mock.py', **storage_kwargs )