def test_validate_pipeline_generator(pipeline, sdc_executor):
bundle = sdc_executor.get_bundle(['PipelineContentGenerator'])
# Manifest must contain the generator
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
assert p.get('com.streamsets.datacollector.bundles.content.PipelineContentGenerator') is not None
assert p.get('com.streamsets.datacollector.bundles.content.LogContentGenerator') is None
assert p.get('com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator') is None
# We should have pipeline in the bundle that we should be able to easily import to the SDC again
with bundle.open(f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/'
f'{pipeline.id}/pipeline.json') as raw:
bundle_json = json.loads(raw.read().decode())
bundle_pipeline = sdc_models.Pipeline(pipeline=bundle_json)
# We need to "reset" the name, otherwise import will fail
bundle_pipeline.id = str(uuid4())
sdc_executor.add_pipeline(bundle_pipeline)
# History have a known structure as the pipeline have not run yet
with bundle.open(f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/'
f'{pipeline.id}/history.json') as raw:
bundle_json = json.loads(raw.read().decode())
bundle_history = sdc_models.History(bundle_json)
assert len(bundle_history) == 1
entry = bundle_history.latest
assert entry['user'] == 'admin'
# Validate existence of some other files
assert (f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/{pipeline.id}/info.json'
in bundle.namelist())
assert (f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/{pipeline.id}/offset.json'
in bundle.namelist())
def test_generate_new_bundle(sdc_executor):
bundle = sdc_executor.get_bundle()
# The manifest is created last and contains all the generators
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
# Default bundle should have the "default" generators
assert p.get(
'com.streamsets.datacollector.bundles.content.PipelineContentGenerator'
) is not None
assert p.get(
'com.streamsets.datacollector.bundles.content.LogContentGenerator'
) is not None
assert p.get(
'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator'
) is not None
# And should have generators that user needs to explicitly allow
assert p.get(
'com.streamsets.datacollector.bundles.content.SnapshotGenerator'
) is None
# Negative case
assert p.get(
'universe.milky_way.solar_system.earth.europe.czech_republic.working_government'
) is None
def test_validate_log_generator(sdc_executor):
bundle = sdc_executor.get_bundle(['LogContentGenerator'])
# Manifest must contain the generator
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
assert p.get(
'com.streamsets.datacollector.bundles.content.LogContentGenerator'
) is not None
assert p.get(
'com.streamsets.datacollector.bundles.content.PipelineContentGenerator'
) is None
assert p.get(
'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator'
) is None
# Main log
with bundle.open(
'com.streamsets.datacollector.bundles.content.LogContentGenerator//sdc.log'
) as raw:
log = raw.read().decode()
assert "Main - Build info" in log
assert "Main - Runtime info" in log
assert "Main - Starting" in log
# We're fine with just validating that gc log is indeed there
assert 'com.streamsets.datacollector.bundles.content.LogContentGenerator//gc.log' in bundle.namelist(
)
def test_validate_snapshot_generator(pipeline, sdc_executor):
generator = 'com.streamsets.datacollector.bundles.content.SnapshotGenerator'
# Generate at least one snapshot
snapshot = sdc_executor.capture_snapshot(pipeline,
start_pipeline=True).snapshot
sdc_executor.stop_pipeline(pipeline)
assert snapshot is not None
bundle = sdc_executor.get_bundle(['SnapshotGenerator'])
# Manifest must contain the generator
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
assert p.get(
'com.streamsets.datacollector.bundles.content.SnapshotGenerator'
) is not None
with bundle.open('{}/{}/{}/output.json'.format(
generator, pipeline.id, snapshot.snapshot_name)) as raw:
bundle_json = json.loads(raw.read().decode())
bundle_snapshot = sdc_models.Snapshot(pipeline.id,
snapshot.snapshot_name,
bundle_json)
assert len(bundle_snapshot) == 1
assert len(
bundle_snapshot[pipeline.origin_stage.instance_name].output) == 10
assert '{}/{}/{}/info.json'.format(
generator, pipeline.id, snapshot.snapshot_name) in bundle.namelist()
def test_validate_redaction(sdc_executor):
bundle = sdc_executor.get_bundle()
# Redaction in files
with bundle.open('com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator/conf/sdc.properties') as raw:
p = Properties()
p.load(raw)
assert p.get('https.keystore.password') == 'REDACTED'
def test_validate_sdc_info_generator(sdc_executor):
bundle = sdc_executor.get_bundle(['SdcInfoContentGenerator'])
bundle_file_root = 'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator'
# Manifest must contain the generator
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
assert p.get(
'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator'
) is not None
assert p.get(
'com.streamsets.datacollector.bundles.content.LogContentGenerator'
) is None
assert p.get(
'com.streamsets.datacollector.bundles.content.PipelineContentGenerator'
) is None
with bundle.open(f'{bundle_file_root}/properties/build.properties') as raw:
p = Properties()
p.load(raw)
assert p.get('version') is not None
with bundle.open(
f'{bundle_file_root}/properties/system.properties') as raw:
p = Properties()
p.load(raw)
assert p.get('os.name') is not None
assert p.get('java.vm.version') is not None
assert p.get('sdc.hostname') is not None
with bundle.open(f'{bundle_file_root}/conf/sdc.properties') as raw:
p = Properties()
p.load(raw)
assert p.get('https.keystore.password') is not None
# We're fine with just validating existence of some other files
assert f'{bundle_file_root}/dir_listing/conf.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/resource.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/data.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/log.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/lib_extra.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/stagelibs.txt' in bundle.namelist()
assert f'{bundle_file_root}/conf/sdc-log4j.properties' in bundle.namelist()
assert f'{bundle_file_root}/conf/dpm.properties' in bundle.namelist()
assert f'{bundle_file_root}/conf/ldap-login.conf' in bundle.namelist()
assert f'{bundle_file_root}/conf/sdc-security.policy' in bundle.namelist()
assert f'{bundle_file_root}/libexec/sdc-env.sh' in bundle.namelist()
assert f'{bundle_file_root}/libexec/sdcd-env.sh' in bundle.namelist()
assert f'{bundle_file_root}/runtime/threads.txt' in bundle.namelist()
assert f'{bundle_file_root}/runtime/jmx.json' in bundle.namelist()
