def test_execute(self, mock_hook):
mock_instance = mock_hook.return_value
operator = WasbDeleteBlobOperator(task_id='wasb_operator',
dag=self.dag,
is_prefix=True,
ignore_if_missing=True,
**self._config)
operator.execute(None)
mock_instance.delete_file.assert_called_once_with(
'container', 'blob', True, True)
def test_init(self):
operator = WasbDeleteBlobOperator(task_id='wasb_operator_1',
dag=self.dag,
**self._config)
assert operator.container_name == self._config['container_name']
assert operator.blob_name == self._config['blob_name']
assert operator.is_prefix is False
assert operator.ignore_if_missing is False
operator = WasbDeleteBlobOperator(task_id='wasb_operator_2',
dag=self.dag,
is_prefix=True,
ignore_if_missing=True,
**self._config)
assert operator.is_prefix is True
assert operator.ignore_if_missing is True
def test_init(self):
operator = WasbDeleteBlobOperator(task_id='wasb_operator',
dag=self.dag,
**self._config)
self.assertEqual(operator.container_name,
self._config['container_name'])
self.assertEqual(operator.blob_name, self._config['blob_name'])
self.assertEqual(operator.is_prefix, False)
self.assertEqual(operator.ignore_if_missing, False)
operator = WasbDeleteBlobOperator(task_id='wasb_operator',
dag=self.dag,
is_prefix=True,
ignore_if_missing=True,
**self._config)
self.assertEqual(operator.is_prefix, True)
self.assertEqual(operator.ignore_if_missing, True)
# under the License.
# Ignore missing args provided by default_args
# type: ignore[call-arg]
import os
from datetime import datetime
from airflow.models import DAG
from airflow.providers.microsoft.azure.operators.wasb_delete_blob import WasbDeleteBlobOperator
from airflow.providers.microsoft.azure.transfers.local_to_wasb import LocalFilesystemToWasbOperator
PATH_TO_UPLOAD_FILE = os.environ.get('AZURE_PATH_TO_UPLOAD_FILE',
'example-text.txt')
with DAG(
"example_local_to_wasb",
schedule_interval="@once",
start_date=datetime(2021, 1, 1),
catchup=False,
default_args={
"container_name": "mycontainer",
"blob_name": "myblob"
},
) as dag:
upload = LocalFilesystemToWasbOperator(task_id="upload_file",
file_path=PATH_TO_UPLOAD_FILE)
delete = WasbDeleteBlobOperator(task_id="delete_file")
upload >> delete
"example_sftp_to_wasb",
schedule_interval=None,
catchup=False,
start_date=datetime(2021, 1, 1), # Override to match your needs
) as dag:
transfer_files_to_sftp_step = SFTPOperator(
task_id="transfer_files_from_local_to_sftp",
local_filepath=FILE_COMPLETE_PATH,
remote_filepath=SFTP_FILE_COMPLETE_PATH,
)
# [START how_to_sftp_to_wasb]
transfer_files_to_azure = SFTPToWasbOperator(
task_id="transfer_files_from_sftp_to_wasb",
# SFTP args
sftp_source_path=SFTP_SRC_PATH,
# AZURE args
container_name=AZURE_CONTAINER_NAME,
blob_prefix=BLOB_PREFIX,
)
# [END how_to_sftp_to_wasb]
delete_blob_file_step = WasbDeleteBlobOperator(
task_id="delete_blob_files",
container_name=AZURE_CONTAINER_NAME,
blob_name=BLOB_PREFIX + SAMPLE_FILENAME,
)
transfer_files_to_sftp_step >> transfer_files_to_azure >> delete_blob_file_step >> delete_sftp_file(
)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
from airflow.models import DAG
from airflow.providers.microsoft.azure.operators.wasb_delete_blob import WasbDeleteBlobOperator
from airflow.providers.microsoft.azure.transfers.local_to_wasb import LocalFilesystemToWasbOperator
from airflow.utils.dates import days_ago
PATH_TO_UPLOAD_FILE = os.environ.get('AZURE_PATH_TO_UPLOAD_FILE',
'example-text.txt')
with DAG("example_local_to_wasb",
schedule_interval="@once",
start_date=days_ago(2)) as dag:
upload = LocalFilesystemToWasbOperator(task_id="upload_file",
file_path=PATH_TO_UPLOAD_FILE,
container_name="mycontainer",
blob_name='myblob')
delete = WasbDeleteBlobOperator(task_id="delete_file",
container_name="mycontainer",
blob_name="myblob")
upload >> delete