def update_source_pipelines(source_: Source):
for pipeline_ in pipeline.repository.get_by_source(source_.name):
try:
sdc_client.update(pipeline_)
except streamsets.manager.StreamsetsException as e:
logger_.debug(str(e), exc_info=True)
continue
logger_.info(f'Pipeline {pipeline_.name} updated')
def test_create_edit_query(self, cli_runner):
input_file_path = get_input_file_path('zabbix_pipelines_edit_query.json')
result = cli_runner.invoke(cli.pipeline.create, ['-f', input_file_path], catch_exceptions=False)
assert result.exit_code == 0
pipeline_ = pipeline.repository.get_by_id('test_zabbix_edit_query')
pipeline_.offset = PipelineOffset(pipeline_.id, '{"version": 2, "offsets": {"": "1611320000_30590"}}', 1611320000.0)
sdc_client.update(pipeline_)
pipeline.repository.save(pipeline_.offset)
def _restore_pipelines():
existing, not_existing = _get_pipelines()
for pipeline_ in not_existing:
click.echo(f'Creating pipeline `{pipeline_.name}`')
pipeline.manager.create(pipeline_)
_update_status(pipeline_)
click.secho('Success', fg='green')
for pipeline_ in existing:
click.echo(f'Updating pipeline `{pipeline_.name}`')
sdc_client.update(pipeline_)
_update_status(pipeline_)
click.secho('Success', fg='green')
def update(pipeline_: Pipeline, config_: dict = None):
with pipeline.repository.SessionManager(pipeline_):
if config_:
_load_config(pipeline_, config_, is_edit=True)
if not pipeline_.config_changed():
logger_.info(f'No need to update pipeline {pipeline_.name}')
return
extra_setup.do(pipeline_)
if pipeline_.uses_schema():
_update_schema(pipeline_)
sdc_client.update(pipeline_)
reset_pipeline_retries(pipeline_)
logger_.info(f'Updated pipeline {pipeline_.name}')
def update(pipeline_id: str):
"""
Update all pipelines configuration, recreate and restart them
"""
pipelines = [pipeline.repository.get_by_id(pipeline_id)
] if pipeline_id else pipeline.repository.get_all()
for p in pipelines:
try:
sdc_client.update(p)
click.secho(f'Pipeline {p.name} updated', fg='green')
except streamsets.manager.StreamsetsException as e:
click.echo(str(e))
continue
def edit(url, update_pipelines=False):
try:
s = streamsets.repository.get_by_url(url)
except streamsets.repository.StreamsetsNotExistsException as e:
raise click.ClickException(str(e))
streamsets_ = _prompt_streamsets(s)
old_external_url = streamsets_.agent_external_url
streamsets_.agent_external_url = _prompt_agent_external_url(streamsets_)
streamsets.repository.save(streamsets_)
if update_pipelines and old_external_url != streamsets_.agent_external_url:
for pipeline_ in pipeline.repository.get_by_streamsets_id(
streamsets_.id):
sdc_client.update(pipeline_)
click.secho('Changes saved', fg='green')
def disable_destination_logs(pipeline_: Pipeline):
pipeline_.destination.disable_logs()
destination.repository.save(pipeline_.destination)
sdc_client.update(pipeline_)