def upload_files(**context):
full_paths, names_only = list_local_files()
for i in range(len(full_paths)):
move_to_gcs = FileToGoogleCloudStorageOperator(task_id='to_gcs',
src=full_paths[i],
dst=names_only[i],
bucket=env.GCS_BUCKET)
logging.info('uploading file ' + names_only[i])
move_to_gcs.execute(context)
os.remove(full_paths[i])
def test_execute(self, mock_hook):
mock_instance = mock_hook.return_value
operator = FileToGoogleCloudStorageOperator(
task_id='gcs_to_file_sensor', dag=self.dag, **self._config)
operator.execute(None)
mock_instance.upload.assert_called_once_with(
bucket_name=self._config['bucket'],
filename=self._config['src'],
gzip=self._config['gzip'],
mime_type=self._config['mime_type'],
object_name=self._config['dst'])
def test_execute(self, mock_hook):
mock_instance = mock_hook.return_value
operator = FileToGoogleCloudStorageOperator(
task_id='gcs_to_file_sensor',
dag=self.dag,
**self._config
)
operator.execute(None)
mock_instance.upload.assert_called_once_with(
bucket=self._config['bucket'],
filename=self._config['src'],
gzip=self._config['gzip'],
mime_type=self._config['mime_type'],
object=self._config['dst']
)