def test_azure_exists(self, monkeypatch):
client = MagicMock(get_blob_properties=MagicMock())
service = MagicMock(get_blob_client=MagicMock(return_value=client))
monkeypatch.setattr(
"prefect.engine.results.azure_result.AzureResult.service", service)
result = AzureResult(container="foo", location="{thing}/here.txt")
assert result.exists("44.txt") is True
def test_azure_init(self, azure_client):
result = AzureResult(container="bob", connection_string="conn")
assert result.value == None
assert result.connection_string == "conn"
assert result.connection_string_secret == None
assert azure_client.called is False
result.initialize_service()
assert azure_client.called is True
def test_azure_writes_binary_string(self, monkeypatch):
client = MagicMock(upload_blob=MagicMock())
service = MagicMock(get_blob_client=MagicMock(return_value=client))
monkeypatch.setattr(
"prefect.engine.results.azure_result.AzureResult.service", service)
result = AzureResult(container="foo", location="nothing/here.txt")
new_result = result.write(None)
assert client.upload_blob.called
assert isinstance(client.upload_blob.call_args[0][0], str)
def test_azure_does_not_exists(self, monkeypatch):
from azure.core.exceptions import ResourceNotFoundError
client = MagicMock(get_blob_properties=MagicMock(
side_effect=ResourceNotFoundError))
service = MagicMock(get_blob_client=MagicMock(return_value=client))
monkeypatch.setattr(
"prefect.engine.results.azure_result.AzureResult.service", service)
result = AzureResult(container="foo", location="{thing}/here.txt")
assert result.exists("44.txt") is False
def test_azure_reads_and_updates_location(self, monkeypatch):
client = MagicMock(download_blob=MagicMock(return_value=""))
service = MagicMock(get_blob_client=MagicMock(return_value=client))
monkeypatch.setattr(
"prefect.engine.results.azure_result.AzureResult.service", service)
result = AzureResult(container="foo", location="{thing}/here.txt")
new_result = result.read("path/to/my/stuff.txt")
assert new_result.location == "path/to/my/stuff.txt"
assert new_result.value is None
def test_azure_writes_to_blob_using_rendered_template_name(self, monkeypatch):
client = MagicMock(upload_blob=MagicMock())
service = MagicMock(get_blob_client=MagicMock(return_value=client))
monkeypatch.setattr(
"prefect.engine.results.azure_result.AzureResult.service", service
)
result = AzureResult(container="foo", location="{thing}/here.txt")
new_result = result.write("so-much-data", thing=42)
assert new_result.location == "42/here.txt"
assert client.upload_blob.called
assert service.get_blob_client.call_args[1]["blob"] == "42/here.txt"
assert service.get_blob_client.call_args[1]["container"] == "foo"
def test_azure_init_connection_string(self, azure_client):
result = AzureResult(container="bob", connection_string="con1")
result.initialize_service()
azure_client.assert_called_with(conn_str="con1")
with prefect.context({"secrets": {"test": "con2"}}):
result = AzureResult(container="bob", connection_string_secret="test")
result.initialize_service()
azure_client.assert_called_with(conn_str="con2")
def __init__(self,
container: str,
connection_string: str = None,
blob_name: str = None,
stored_as_script: bool = False,
**kwargs: Any) -> None:
self.connection_string = connection_string or os.getenv(
"AZURE_STORAGE_CONNECTION_STRING")
self.container = container
self.blob_name = blob_name
result = AzureResult(connection_string=self.connection_string,
container=container)
super().__init__(result=result,
stored_as_script=stored_as_script,
**kwargs)
def __init__(self,
container: str,
connection_string: str = None,
blob_name: str = None,
**kwargs: Any) -> None:
self.flows = dict() # type: Dict[str, str]
self._flows = dict() # type: Dict[str, "Flow"]
self.connection_string = connection_string or os.getenv(
"AZURE_STORAGE_CONNECTION_STRING")
self.container = container
self.blob_name = blob_name
result = AzureResult(connection_string=self.connection_string,
container=container)
super().__init__(result=result, **kwargs)
def __init__(
self,
container: str,
connection_string_secret: str = None,
blob_name: str = None,
overwrite: bool = False,
stored_as_script: bool = False,
**kwargs: Any
) -> None:
self.container = container
self.connection_string_secret = connection_string_secret
self.blob_name = blob_name
self.overwrite = overwrite
result = AzureResult(
connection_string_secret=self.connection_string_secret,
container=container,
)
super().__init__(result=result, stored_as_script=stored_as_script, **kwargs)
def test_azure_result_is_pickleable(self, azure_client):
result = AzureResult("foo")
res = cloudpickle.loads(cloudpickle.dumps(result))
assert isinstance(res, AzureResult)
def test_azure_init_with_values(self, azure_client):
result = AzureResult(container="bob", connection_string="conn", value=3)
assert result.value == 3
from prefect.engine.results import AzureResult
import os
# con_string = ""
azure_result = AzureResult(container="flows",
connection_string=os.environ["AZ"],
location="test5")
# w = azure_result.write({"Test": "123"})
r = azure_result.read("test5")
print(r)
def test_azure_init_without_connection_string(self, azure_client):
result = AzureResult(container="bob", connection_string=None)
with pytest.raises(ValueError):
result.initialize_service()
# Storage
FLOWS_DIR_PATH = '/opt/server/src/flows'
storage_kwargs = {
'dockerfile': 'server/Dockerfile',
'registry_url': REGISTRY_URL,
'stored_as_script': True,
}
# Executer
local_executor = LocalExecutor()
dask_executor = DaskExecutor(address=DASK_SCHEDULER_ADDR)
# Result
if RESULT_SUBCLASS == 'azure':
result = AzureResult(container=AZURE_RESULT_CONTAINER)
elif RESULT_SUBCLASS == 's3':
result = S3Result(bucket=S3_RESULT_BUCKET)
else:
result = LocalResult(dir=LOCAL_RESULT_DIR)
# Set flow run configs
mapreduce_wordcount.run_config = run_config
# Set flow storage
mapreduce_wordcount.storage = Docker(
path=f'{FLOWS_DIR_PATH}/mock.py',
**storage_kwargs
)
