def upload_files(**context): full_paths, names_only = list_local_files() for i in range(len(full_paths)): move_to_gcs = FileToGoogleCloudStorageOperator(task_id='to_gcs', src=full_paths[i], dst=names_only[i], bucket=env.GCS_BUCKET) logging.info('uploading file ' + names_only[i]) move_to_gcs.execute(context) os.remove(full_paths[i])
def test_execute(self, mock_hook): mock_instance = mock_hook.return_value operator = FileToGoogleCloudStorageOperator( task_id='gcs_to_file_sensor', dag=self.dag, **self._config) operator.execute(None) mock_instance.upload.assert_called_once_with( bucket_name=self._config['bucket'], filename=self._config['src'], gzip=self._config['gzip'], mime_type=self._config['mime_type'], object_name=self._config['dst'])
def test_execute(self, mock_hook): mock_instance = mock_hook.return_value operator = FileToGoogleCloudStorageOperator( task_id='gcs_to_file_sensor', dag=self.dag, **self._config ) operator.execute(None) mock_instance.upload.assert_called_once_with( bucket=self._config['bucket'], filename=self._config['src'], gzip=self._config['gzip'], mime_type=self._config['mime_type'], object=self._config['dst'] )