def test_propclass_getitem():
p = Properties()
p.load(StringIO(INPUT))
assert p["key"] == "value"
assert p["foo"] == "second definition"
with pytest.raises(KeyError):
p["missing"]
def test_propclass_eq_repeated_keys():
p = Properties()
p.load(StringIO('key = value\nkey: other value\n'))
p2 = Properties()
p2.load(StringIO('key: whatever\nkey other value'))
assert p == p2
assert dict(p) == dict(p2) == {"key": "other value"}
def test_propclass_copy_more():
p = Properties()
p.load(StringIO(INPUT))
p2 = p.copy()
assert p is not p2
assert isinstance(p2, Properties)
assert p == p2
assert dict(p) == dict(p2) == {
"foo": "second definition",
"bar": "only definition",
"key": "value",
"zebra": "apple",
}
p2["foo"] = "third definition"
del p2["bar"]
p2["key"] = "value"
p2["zebra"] = "horse"
p2["new"] = "old"
assert p != p2
assert dict(p) == {
"foo": "second definition",
"bar": "only definition",
"key": "value",
"zebra": "apple",
}
assert dict(p2) == {
"foo": "third definition",
"key": "value",
"zebra": "horse",
"new": "old",
}
def test_validate_pipeline_generator(pipeline, sdc_executor):
bundle = sdc_executor.get_bundle(['PipelineContentGenerator'])
# Manifest must contain the generator
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
assert p.get('com.streamsets.datacollector.bundles.content.PipelineContentGenerator') is not None
assert p.get('com.streamsets.datacollector.bundles.content.LogContentGenerator') is None
assert p.get('com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator') is None
# We should have pipeline in the bundle that we should be able to easily import to the SDC again
with bundle.open(f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/'
f'{pipeline.id}/pipeline.json') as raw:
bundle_json = json.loads(raw.read().decode())
bundle_pipeline = sdc_models.Pipeline(pipeline=bundle_json)
# We need to "reset" the name, otherwise import will fail
bundle_pipeline.id = str(uuid4())
sdc_executor.add_pipeline(bundle_pipeline)
# History have a known structure as the pipeline have not run yet
with bundle.open(f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/'
f'{pipeline.id}/history.json') as raw:
bundle_json = json.loads(raw.read().decode())
bundle_history = sdc_models.History(bundle_json)
assert len(bundle_history) == 1
entry = bundle_history.latest
assert entry['user'] == 'admin'
# Validate existence of some other files
assert (f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/{pipeline.id}/info.json'
in bundle.namelist())
assert (f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/{pipeline.id}/offset.json'
in bundle.namelist())
def test_generate_new_bundle(sdc_executor):
bundle = sdc_executor.get_bundle()
# The manifest is created last and contains all the generators
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
# Default bundle should have the "default" generators
assert p.get(
'com.streamsets.datacollector.bundles.content.PipelineContentGenerator'
) is not None
assert p.get(
'com.streamsets.datacollector.bundles.content.LogContentGenerator'
) is not None
assert p.get(
'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator'
) is not None
# And should have generators that user needs to explicitly allow
assert p.get(
'com.streamsets.datacollector.bundles.content.SnapshotGenerator'
) is None
# Negative case
assert p.get(
'universe.milky_way.solar_system.earth.europe.czech_republic.working_government'
) is None
def test_validate_snapshot_generator(pipeline, sdc_executor):
generator = 'com.streamsets.datacollector.bundles.content.SnapshotGenerator'
# Generate at least one snapshot
snapshot = sdc_executor.capture_snapshot(pipeline,
start_pipeline=True).snapshot
sdc_executor.stop_pipeline(pipeline)
assert snapshot is not None
bundle = sdc_executor.get_bundle(['SnapshotGenerator'])
# Manifest must contain the generator
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
assert p.get(
'com.streamsets.datacollector.bundles.content.SnapshotGenerator'
) is not None
with bundle.open('{}/{}/{}/output.json'.format(
generator, pipeline.id, snapshot.snapshot_name)) as raw:
bundle_json = json.loads(raw.read().decode())
bundle_snapshot = sdc_models.Snapshot(pipeline.id,
snapshot.snapshot_name,
bundle_json)
assert len(bundle_snapshot) == 1
assert len(
bundle_snapshot[pipeline.origin_stage.instance_name].output) == 10
assert '{}/{}/{}/info.json'.format(
generator, pipeline.id, snapshot.snapshot_name) in bundle.namelist()
def test_propclass_eq_different_comments():
p = Properties()
p.load(StringIO('#This is a comment.\nkey=value\n'))
p2 = Properties()
p2.load(StringIO('#This is also a comment.\nkey=value\n'))
assert p == p2
assert dict(p) == dict(p2)
def test_validate_log_generator(sdc_executor):
bundle = sdc_executor.get_bundle(['LogContentGenerator'])
# Manifest must contain the generator
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
assert p.get(
'com.streamsets.datacollector.bundles.content.LogContentGenerator'
) is not None
assert p.get(
'com.streamsets.datacollector.bundles.content.PipelineContentGenerator'
) is None
assert p.get(
'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator'
) is None
# Main log
with bundle.open(
'com.streamsets.datacollector.bundles.content.LogContentGenerator//sdc.log'
) as raw:
log = raw.read().decode()
assert "Main - Build info" in log
assert "Main - Runtime info" in log
assert "Main - Starting" in log
# We're fine with just validating that gc log is indeed there
assert 'com.streamsets.datacollector.bundles.content.LogContentGenerator//gc.log' in bundle.namelist(
)
def test_validate_redaction(sdc_executor):
bundle = sdc_executor.get_bundle()
# Redaction in files
with bundle.open('com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator/conf/sdc.properties') as raw:
p = Properties()
p.load(raw)
assert p.get('https.keystore.password') == 'REDACTED'
def test_propclass_load_eq_from_dict():
p = Properties()
p.load(StringIO(INPUT))
assert p == Properties({
"foo": "second definition",
"bar": "only definition",
"key": "value",
"zebra": "apple",
})
def test_propclass_eq_set_nochange():
p = Properties()
p.load(StringIO(INPUT))
p2 = Properties()
p2.load(StringIO(INPUT))
assert p == p2
assert p["key"] == p2["key"] == "value"
p2["key"] = "value"
assert p == p2
assert dict(p) == dict(p2)
def test_propclass_load():
p = Properties()
p.load(StringIO(INPUT))
assert len(p) == 4
assert bool(p)
assert dict(p) == {
"foo": "second definition",
"bar": "only definition",
"key": "value",
"zebra": "apple",
}
def get_all_stage_libs():
raw_stagelibs = urllib.request.urlopen(
"http://nightly.streamsets.com.s3-us-west-2.amazonaws.com/datacollector/latest/tarball/stage-lib-manifest.properties"
)
p = Properties()
p.load(raw_stagelibs)
return [
lib for lib in
[lib.replace('stage-lib.', '') for lib in p if 'stage-lib.' in lib]
if lib not in EXCLUDE_LIBS
]
def test_propclass_delitem():
p = Properties()
p.load(StringIO(INPUT))
del p["key"]
assert len(p) == 3
assert bool(p)
assert dict(p) == {
"foo": "second definition",
"bar": "only definition",
"zebra": "apple",
}
def test_propclass_setitem():
p = Properties()
p.load(StringIO(INPUT))
p["key"] = "lock"
assert len(p) == 4
assert bool(p)
assert dict(p) == {
"foo": "second definition",
"bar": "only definition",
"key": "lock",
"zebra": "apple",
}
def test_propclass_nonempty_load():
p = Properties({"key": "lock", "horse": "orange"})
p.load(StringIO(INPUT))
assert len(p) == 5
assert bool(p)
assert dict(p) == {
"foo": "second definition",
"bar": "only definition",
"horse": "orange",
"key": "value",
"zebra": "apple",
}
def test_propclass_delitem_missing():
p = Properties()
p.load(StringIO(INPUT))
with pytest.raises(KeyError):
del p["missing"]
assert len(p) == 4
assert bool(p)
assert dict(p) == {
"foo": "second definition",
"bar": "only definition",
"key": "value",
"zebra": "apple",
}
def test_propclass_additem():
p = Properties()
p.load(StringIO(INPUT))
p["new"] = "old"
assert len(p) == 5
assert bool(p)
assert dict(p) == {
"foo": "second definition",
"bar": "only definition",
"key": "value",
"zebra": "apple",
"new": "old",
}
def updatePropertyFile(path, stcm, mavenPath=''):
jp = Properties()
jp.load(open(osp.join(stcm, 'paths.properties')))
newProperties = {}
for k, v in jp.items():
if k == 'mavenHome':
if not mavenPath == '':
newProperties[k] = mavenPath
elif k == 'PathToSetup':
newProperties[k] = path
else:
newProperties[k] = v
with open(osp.join(stcm, 'paths.properties'), 'w') as f:
javaproperties.dump(newProperties, f)
def test_validate_sdc_info_generator(sdc_executor):
bundle = sdc_executor.get_bundle(['SdcInfoContentGenerator'])
bundle_file_root = 'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator'
# Manifest must contain the generator
with bundle.open('generators.properties') as zip_file:
p = Properties()
p.load(zip_file)
assert p.get(
'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator'
) is not None
assert p.get(
'com.streamsets.datacollector.bundles.content.LogContentGenerator'
) is None
assert p.get(
'com.streamsets.datacollector.bundles.content.PipelineContentGenerator'
) is None
with bundle.open(f'{bundle_file_root}/properties/build.properties') as raw:
p = Properties()
p.load(raw)
assert p.get('version') is not None
with bundle.open(
f'{bundle_file_root}/properties/system.properties') as raw:
p = Properties()
p.load(raw)
assert p.get('os.name') is not None
assert p.get('java.vm.version') is not None
assert p.get('sdc.hostname') is not None
with bundle.open(f'{bundle_file_root}/conf/sdc.properties') as raw:
p = Properties()
p.load(raw)
assert p.get('https.keystore.password') is not None
# We're fine with just validating existence of some other files
assert f'{bundle_file_root}/dir_listing/conf.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/resource.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/data.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/log.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/lib_extra.txt' in bundle.namelist()
assert f'{bundle_file_root}/dir_listing/stagelibs.txt' in bundle.namelist()
assert f'{bundle_file_root}/conf/sdc-log4j.properties' in bundle.namelist()
assert f'{bundle_file_root}/conf/dpm.properties' in bundle.namelist()
assert f'{bundle_file_root}/conf/ldap-login.conf' in bundle.namelist()
assert f'{bundle_file_root}/conf/sdc-security.policy' in bundle.namelist()
assert f'{bundle_file_root}/libexec/sdc-env.sh' in bundle.namelist()
assert f'{bundle_file_root}/libexec/sdcd-env.sh' in bundle.namelist()
assert f'{bundle_file_root}/runtime/threads.txt' in bundle.namelist()
assert f'{bundle_file_root}/runtime/jmx.json' in bundle.namelist()
def test_propclass_eq_self():
p = Properties()
p.load(StringIO(INPUT))
assert p == p
def test_propclass_neq_string():
p = Properties()
p.load(StringIO(INPUT))
assert p != INPUT
assert INPUT != p
