def test_propclass_getitem(): p = Properties() p.load(StringIO(INPUT)) assert p["key"] == "value" assert p["foo"] == "second definition" with pytest.raises(KeyError): p["missing"]
def test_propclass_eq_repeated_keys(): p = Properties() p.load(StringIO('key = value\nkey: other value\n')) p2 = Properties() p2.load(StringIO('key: whatever\nkey other value')) assert p == p2 assert dict(p) == dict(p2) == {"key": "other value"}
def test_propclass_copy_more(): p = Properties() p.load(StringIO(INPUT)) p2 = p.copy() assert p is not p2 assert isinstance(p2, Properties) assert p == p2 assert dict(p) == dict(p2) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", } p2["foo"] = "third definition" del p2["bar"] p2["key"] = "value" p2["zebra"] = "horse" p2["new"] = "old" assert p != p2 assert dict(p) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", } assert dict(p2) == { "foo": "third definition", "key": "value", "zebra": "horse", "new": "old", }
def test_validate_pipeline_generator(pipeline, sdc_executor): bundle = sdc_executor.get_bundle(['PipelineContentGenerator']) # Manifest must contain the generator with bundle.open('generators.properties') as zip_file: p = Properties() p.load(zip_file) assert p.get('com.streamsets.datacollector.bundles.content.PipelineContentGenerator') is not None assert p.get('com.streamsets.datacollector.bundles.content.LogContentGenerator') is None assert p.get('com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator') is None # We should have pipeline in the bundle that we should be able to easily import to the SDC again with bundle.open(f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/' f'{pipeline.id}/pipeline.json') as raw: bundle_json = json.loads(raw.read().decode()) bundle_pipeline = sdc_models.Pipeline(pipeline=bundle_json) # We need to "reset" the name, otherwise import will fail bundle_pipeline.id = str(uuid4()) sdc_executor.add_pipeline(bundle_pipeline) # History have a known structure as the pipeline have not run yet with bundle.open(f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/' f'{pipeline.id}/history.json') as raw: bundle_json = json.loads(raw.read().decode()) bundle_history = sdc_models.History(bundle_json) assert len(bundle_history) == 1 entry = bundle_history.latest assert entry['user'] == 'admin' # Validate existence of some other files assert (f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/{pipeline.id}/info.json' in bundle.namelist()) assert (f'com.streamsets.datacollector.bundles.content.PipelineContentGenerator/{pipeline.id}/offset.json' in bundle.namelist())
def test_generate_new_bundle(sdc_executor): bundle = sdc_executor.get_bundle() # The manifest is created last and contains all the generators with bundle.open('generators.properties') as zip_file: p = Properties() p.load(zip_file) # Default bundle should have the "default" generators assert p.get( 'com.streamsets.datacollector.bundles.content.PipelineContentGenerator' ) is not None assert p.get( 'com.streamsets.datacollector.bundles.content.LogContentGenerator' ) is not None assert p.get( 'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator' ) is not None # And should have generators that user needs to explicitly allow assert p.get( 'com.streamsets.datacollector.bundles.content.SnapshotGenerator' ) is None # Negative case assert p.get( 'universe.milky_way.solar_system.earth.europe.czech_republic.working_government' ) is None
def test_validate_snapshot_generator(pipeline, sdc_executor): generator = 'com.streamsets.datacollector.bundles.content.SnapshotGenerator' # Generate at least one snapshot snapshot = sdc_executor.capture_snapshot(pipeline, start_pipeline=True).snapshot sdc_executor.stop_pipeline(pipeline) assert snapshot is not None bundle = sdc_executor.get_bundle(['SnapshotGenerator']) # Manifest must contain the generator with bundle.open('generators.properties') as zip_file: p = Properties() p.load(zip_file) assert p.get( 'com.streamsets.datacollector.bundles.content.SnapshotGenerator' ) is not None with bundle.open('{}/{}/{}/output.json'.format( generator, pipeline.id, snapshot.snapshot_name)) as raw: bundle_json = json.loads(raw.read().decode()) bundle_snapshot = sdc_models.Snapshot(pipeline.id, snapshot.snapshot_name, bundle_json) assert len(bundle_snapshot) == 1 assert len( bundle_snapshot[pipeline.origin_stage.instance_name].output) == 10 assert '{}/{}/{}/info.json'.format( generator, pipeline.id, snapshot.snapshot_name) in bundle.namelist()
def test_propclass_eq_different_comments(): p = Properties() p.load(StringIO('#This is a comment.\nkey=value\n')) p2 = Properties() p2.load(StringIO('#This is also a comment.\nkey=value\n')) assert p == p2 assert dict(p) == dict(p2)
def test_validate_log_generator(sdc_executor): bundle = sdc_executor.get_bundle(['LogContentGenerator']) # Manifest must contain the generator with bundle.open('generators.properties') as zip_file: p = Properties() p.load(zip_file) assert p.get( 'com.streamsets.datacollector.bundles.content.LogContentGenerator' ) is not None assert p.get( 'com.streamsets.datacollector.bundles.content.PipelineContentGenerator' ) is None assert p.get( 'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator' ) is None # Main log with bundle.open( 'com.streamsets.datacollector.bundles.content.LogContentGenerator//sdc.log' ) as raw: log = raw.read().decode() assert "Main - Build info" in log assert "Main - Runtime info" in log assert "Main - Starting" in log # We're fine with just validating that gc log is indeed there assert 'com.streamsets.datacollector.bundles.content.LogContentGenerator//gc.log' in bundle.namelist( )
def test_validate_redaction(sdc_executor): bundle = sdc_executor.get_bundle() # Redaction in files with bundle.open('com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator/conf/sdc.properties') as raw: p = Properties() p.load(raw) assert p.get('https.keystore.password') == 'REDACTED'
def test_propclass_load_eq_from_dict(): p = Properties() p.load(StringIO(INPUT)) assert p == Properties({ "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", })
def test_propclass_eq_set_nochange(): p = Properties() p.load(StringIO(INPUT)) p2 = Properties() p2.load(StringIO(INPUT)) assert p == p2 assert p["key"] == p2["key"] == "value" p2["key"] = "value" assert p == p2 assert dict(p) == dict(p2)
def test_propclass_load(): p = Properties() p.load(StringIO(INPUT)) assert len(p) == 4 assert bool(p) assert dict(p) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", }
def get_all_stage_libs(): raw_stagelibs = urllib.request.urlopen( "http://nightly.streamsets.com.s3-us-west-2.amazonaws.com/datacollector/latest/tarball/stage-lib-manifest.properties" ) p = Properties() p.load(raw_stagelibs) return [ lib for lib in [lib.replace('stage-lib.', '') for lib in p if 'stage-lib.' in lib] if lib not in EXCLUDE_LIBS ]
def test_propclass_delitem(): p = Properties() p.load(StringIO(INPUT)) del p["key"] assert len(p) == 3 assert bool(p) assert dict(p) == { "foo": "second definition", "bar": "only definition", "zebra": "apple", }
def test_propclass_setitem(): p = Properties() p.load(StringIO(INPUT)) p["key"] = "lock" assert len(p) == 4 assert bool(p) assert dict(p) == { "foo": "second definition", "bar": "only definition", "key": "lock", "zebra": "apple", }
def test_propclass_nonempty_load(): p = Properties({"key": "lock", "horse": "orange"}) p.load(StringIO(INPUT)) assert len(p) == 5 assert bool(p) assert dict(p) == { "foo": "second definition", "bar": "only definition", "horse": "orange", "key": "value", "zebra": "apple", }
def test_propclass_delitem_missing(): p = Properties() p.load(StringIO(INPUT)) with pytest.raises(KeyError): del p["missing"] assert len(p) == 4 assert bool(p) assert dict(p) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", }
def test_propclass_additem(): p = Properties() p.load(StringIO(INPUT)) p["new"] = "old" assert len(p) == 5 assert bool(p) assert dict(p) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", "new": "old", }
def updatePropertyFile(path, stcm, mavenPath=''): jp = Properties() jp.load(open(osp.join(stcm, 'paths.properties'))) newProperties = {} for k, v in jp.items(): if k == 'mavenHome': if not mavenPath == '': newProperties[k] = mavenPath elif k == 'PathToSetup': newProperties[k] = path else: newProperties[k] = v with open(osp.join(stcm, 'paths.properties'), 'w') as f: javaproperties.dump(newProperties, f)
def test_validate_sdc_info_generator(sdc_executor): bundle = sdc_executor.get_bundle(['SdcInfoContentGenerator']) bundle_file_root = 'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator' # Manifest must contain the generator with bundle.open('generators.properties') as zip_file: p = Properties() p.load(zip_file) assert p.get( 'com.streamsets.datacollector.bundles.content.SdcInfoContentGenerator' ) is not None assert p.get( 'com.streamsets.datacollector.bundles.content.LogContentGenerator' ) is None assert p.get( 'com.streamsets.datacollector.bundles.content.PipelineContentGenerator' ) is None with bundle.open(f'{bundle_file_root}/properties/build.properties') as raw: p = Properties() p.load(raw) assert p.get('version') is not None with bundle.open( f'{bundle_file_root}/properties/system.properties') as raw: p = Properties() p.load(raw) assert p.get('os.name') is not None assert p.get('java.vm.version') is not None assert p.get('sdc.hostname') is not None with bundle.open(f'{bundle_file_root}/conf/sdc.properties') as raw: p = Properties() p.load(raw) assert p.get('https.keystore.password') is not None # We're fine with just validating existence of some other files assert f'{bundle_file_root}/dir_listing/conf.txt' in bundle.namelist() assert f'{bundle_file_root}/dir_listing/resource.txt' in bundle.namelist() assert f'{bundle_file_root}/dir_listing/data.txt' in bundle.namelist() assert f'{bundle_file_root}/dir_listing/log.txt' in bundle.namelist() assert f'{bundle_file_root}/dir_listing/lib_extra.txt' in bundle.namelist() assert f'{bundle_file_root}/dir_listing/stagelibs.txt' in bundle.namelist() assert f'{bundle_file_root}/conf/sdc-log4j.properties' in bundle.namelist() assert f'{bundle_file_root}/conf/dpm.properties' in bundle.namelist() assert f'{bundle_file_root}/conf/ldap-login.conf' in bundle.namelist() assert f'{bundle_file_root}/conf/sdc-security.policy' in bundle.namelist() assert f'{bundle_file_root}/libexec/sdc-env.sh' in bundle.namelist() assert f'{bundle_file_root}/libexec/sdcd-env.sh' in bundle.namelist() assert f'{bundle_file_root}/runtime/threads.txt' in bundle.namelist() assert f'{bundle_file_root}/runtime/jmx.json' in bundle.namelist()
def test_propclass_eq_self(): p = Properties() p.load(StringIO(INPUT)) assert p == p
def test_propclass_neq_string(): p = Properties() p.load(StringIO(INPUT)) assert p != INPUT assert INPUT != p