def test_poke(self, mock_hook): mock_instance = mock_hook.return_value sensor = WasbBlobSensor( task_id='wasb_sensor', dag=self.dag, check_options={'timeout': 2}, **self._config ) sensor.poke(None) mock_instance.check_for_blob.assert_called_once_with('container', 'blob', timeout=2)
def test_init(self): sensor = WasbBlobSensor(task_id='wasb_sensor_1', dag=self.dag, **self._config) self.assertEqual(sensor.container_name, self._config['container_name']) self.assertEqual(sensor.blob_name, self._config['blob_name']) self.assertEqual(sensor.wasb_conn_id, self._config['wasb_conn_id']) self.assertEqual(sensor.check_options, {}) self.assertEqual(sensor.timeout, self._config['timeout']) sensor = WasbBlobSensor( task_id='wasb_sensor_2', dag=self.dag, check_options={'timeout': 2}, **self._config ) self.assertEqual(sensor.check_options, {'timeout': 2})
def test_init(self): sensor = WasbBlobSensor(task_id='wasb_sensor_1', dag=self.dag, **self._config) assert sensor.container_name == self._config['container_name'] assert sensor.blob_name == self._config['blob_name'] assert sensor.wasb_conn_id == self._config['wasb_conn_id'] assert sensor.check_options == {} assert sensor.timeout == self._config['timeout'] sensor = WasbBlobSensor(task_id='wasb_sensor_2', dag=self.dag, check_options={'timeout': 2}, **self._config) assert sensor.check_options == {'timeout': 2}
BLOB_NAME = os.environ.get("AZURE_BLOB_NAME", "file.txt") AZURE_CONTAINER_NAME = os.environ.get("AZURE_CONTAINER_NAME", "airflow") GCP_BUCKET_FILE_PATH = os.environ.get("GCP_BUCKET_FILE_PATH", "file.txt") GCP_BUCKET_NAME = os.environ.get("GCP_BUCKET_NAME", "azure_bucket") GCP_OBJECT_NAME = os.environ.get("GCP_OBJECT_NAME", "file.txt") with DAG( "example_azure_blob_to_gcs", schedule_interval=None, start_date=days_ago(1), # Override to match your needs ) as dag: # [START how_to_wait_for_blob] wait_for_blob = WasbBlobSensor( task_id="wait_for_blob", wasb_conn_id="wasb_default", container_name=AZURE_CONTAINER_NAME, blob_name=BLOB_NAME, ) # [END how_to_wait_for_blob] # [START how_to_azure_blob_to_gcs] transfer_files_to_gcs = AzureBlobStorageToGCSOperator( task_id="transfer_files_to_gcs", # AZURE args wasb_conn_id="wasb_default", container_name=AZURE_CONTAINER_NAME, blob_name=BLOB_NAME, file_path=GCP_OBJECT_NAME, # GCP args gcp_conn_id="google_cloud_default", bucket_name=GCP_BUCKET_NAME,
GCP_BUCKET_FILE_PATH = os.environ.get("GCP_BUCKET_FILE_PATH", "file.txt") GCP_BUCKET_NAME = os.environ.get("GCP_BUCKET_NAME", "INVALID BUCKET NAME") GCP_OBJECT_NAME = os.environ.get("GCP_OBJECT_NAME", "file.txt") # [START how_to_azure_blob_to_gcs] with DAG( "example_azure_blob_to_gcs", schedule_interval=None, start_date=datetime(2021, 1, 1), # Override to match your needs default_args={ "container_name": AZURE_CONTAINER_NAME, "blob_name": BLOB_NAME }, ) as dag: wait_for_blob = WasbBlobSensor(task_id="wait_for_blob") transfer_files_to_gcs = AzureBlobStorageToGCSOperator( task_id="transfer_files_to_gcs", # AZURE arg file_path=GCP_OBJECT_NAME, # GCP args bucket_name=GCP_BUCKET_NAME, object_name=GCP_OBJECT_NAME, filename=GCP_BUCKET_FILE_PATH, gzip=False, delegate_to=None, impersonation_chain=None, ) # [END how_to_azure_blob_to_gcs]