def test_propfile_eq_set_nochange(): pf = PropertiesFile.loads(INPUT) pf2 = PropertiesFile.loads(INPUT) assert pf == pf2 assert pf["key"] == pf2["key"] == "value" pf2["key"] = "value" assert pf == pf2 assert dict(pf) == dict(pf2) assert pf.dumps() == INPUT assert pf.dumps() != pf2.dumps()
def test_propfile_add_after_no_trailing_newline(): pf = PropertiesFile.loads('key = value\\') pf._check() pf["new"] = "old" pf._check() assert dict(pf) == {"key": "value", "new": "old"} assert pf.dumps() == 'key = value\nnew=old\n'
def test_propfile_add_after_trailing_comment_escape_nl(): pf = PropertiesFile.loads('#key = value\\\n') pf._check() pf["new"] = "old" pf._check() assert dict(pf) == {"new": "old"} assert pf.dumps() == '#key = value\\\nnew=old\n'
def test_propfile_delete_header_comment(part1, part2): pf = PropertiesFile.loads(part1 + part2) pf._check() del pf.header_comment pf._check() assert pf.header_comment is None assert pf.dumps() == part2
def test_propfile_set_header_comment(part1, part2, c, c2, csrc): pf = PropertiesFile.loads(part1 + part2) pf._check() pf.header_comment = c pf._check() assert pf.header_comment == c2 assert pf.dumps() == csrc + part2
def test_propfile_delete_timestamp(src, ts2, result): pf = PropertiesFile.loads(src) pf._check() del pf.timestamp pf._check() assert pf.timestamp == ts2 assert pf.dumps() == result
def test_propfile_set_timestamp_now(fixed_timestamp): pf = PropertiesFile.loads('key=value\n') pf._check() pf.timestamp = True pf._check() assert pf.timestamp == fixed_timestamp assert pf.dumps() == '#' + fixed_timestamp + '\nkey=value\n'
def test_propfile_delete_repeated_key(): pf = PropertiesFile.loads(INPUT) pf._check() del pf["foo"] pf._check() assert list(pf) == ["bar", "key", "zebra"] assert list(reversed(pf)) == ["zebra", "key", "bar"] assert pf.dumps() == '''\
def test_propfile_set_repeated_key(): pf = PropertiesFile.loads(INPUT) pf._check() pf["foo"] = "redefinition" pf._check() assert list(pf) == ["foo", "bar", "key", "zebra"] assert list(reversed(pf)) == ["zebra", "key", "bar", "foo"] assert pf.dumps() == '''\
def test_propfile_getitem(): pf = PropertiesFile.loads(INPUT) pf._check() assert pf["key"] == "value" assert pf["foo"] == "second definition" with pytest.raises(KeyError): pf["missing"] pf._check()
def test_propfile_additem(): pf = PropertiesFile.loads(INPUT) pf._check() pf["new"] = "old" pf._check() assert list(pf) == ["foo", "bar", "key", "zebra", "new"] assert list(reversed(pf)) == ["new", "zebra", "key", "bar", "foo"] assert pf.dumps() == '''\
def test_propfile_set_nochange(): pf = PropertiesFile.loads(INPUT) pf._check() assert pf["key"] == "value" pf["key"] = "value" pf._check() assert list(pf) == ["foo", "bar", "key", "zebra"] assert list(reversed(pf)) == ["zebra", "key", "bar", "foo"] assert pf.dumps() == '''\
def test_propfile_to_ordereddict(): pf = PropertiesFile.loads(INPUT) pf._check() assert OrderedDict(pf) == OrderedDict([ ("foo", "second definition"), ("bar", "only definition"), ("key", "value"), ("zebra", "apple"), ])
def test_propfile_move_item(): pf = PropertiesFile.loads(INPUT) pf._check() del pf["key"] pf._check() pf["key"] = "recreated" pf._check() assert list(pf) == ["foo", "bar", "zebra", "key"] assert list(reversed(pf)) == ["key", "zebra", "bar", "foo"] assert pf.dumps() == '''\
def test_propfile_delete_repeated_key(): pf = PropertiesFile.loads(INPUT) pf._check() del pf["foo"] pf._check() assert dict(pf) == { "bar": "only definition", "key": "value", "zebra": "apple", } assert list(pf) == ["bar", "key", "zebra"] assert list(reversed(pf)) == ["zebra", "key", "bar"] assert pf.dumps() == '''\
def test_propfile_loads(): pf = PropertiesFile.loads(INPUT) pf._check() assert len(pf) == 4 assert bool(pf) assert dict(pf) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", } assert list(pf) == ["foo", "bar", "key", "zebra"] assert list(reversed(pf)) == ["zebra", "key", "bar", "foo"]
def test_propfile_delitem(): pf = PropertiesFile.loads(INPUT) pf._check() del pf["key"] pf._check() assert dict(pf) == { "foo": "second definition", "bar": "only definition", "zebra": "apple", } assert list(pf) == ["foo", "bar", "zebra"] assert list(reversed(pf)) == ["zebra", "bar", "foo"] assert pf.dumps() == '''\
def test_propfile_additem(): pf = PropertiesFile.loads(INPUT) pf._check() pf["new"] = "old" pf._check() assert dict(pf) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", "new": "old", } assert list(pf) == ["foo", "bar", "key", "zebra", "new"] assert list(reversed(pf)) == ["new", "zebra", "key", "bar", "foo"] assert pf.dumps() == '''\
def test_propfile_set_nochange(): pf = PropertiesFile.loads(INPUT) pf._check() assert pf["key"] == "value" pf["key"] = "value" pf._check() assert dict(pf) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", } assert list(pf) == ["foo", "bar", "key", "zebra"] assert list(reversed(pf)) == ["zebra", "key", "bar", "foo"] assert pf.dumps() == '''\
def test_propfile_delitem_missing(): pf = PropertiesFile.loads(INPUT) pf._check() with pytest.raises(KeyError): del pf["missing"] pf._check() assert len(pf) == 4 assert bool(pf) assert dict(pf) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", } assert list(pf) == ["foo", "bar", "key", "zebra"] assert list(reversed(pf)) == ["zebra", "key", "bar", "foo"] assert pf.dumps() == INPUT
def test_propfile_copy_more(): pf = PropertiesFile.loads(INPUT) pf2 = pf.copy() pf._check() pf2._check() assert pf is not pf2 assert isinstance(pf2, PropertiesFile) assert pf == pf2 assert dict(pf) == dict(pf2) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", } pf2["foo"] = "third definition" del pf2["bar"] pf2["key"] = "value" pf2["zebra"] = "horse" pf2["new"] = "old" pf._check() pf2._check() assert pf != pf2 assert dict(pf) == { "foo": "second definition", "bar": "only definition", "key": "value", "zebra": "apple", } assert dict(pf2) == { "foo": "third definition", "key": "value", "zebra": "horse", "new": "old", } assert pf.dumps() == INPUT assert pf2.dumps() == '''\
def main(args): quiet = not args.verbose node_image = '{}/{}/topology_apache_pulsar:pulsar-{}'.format(args.registry, args.namespace or DEFAULT_NAMESPACE, args.pulsar_version) ports = [{WEB_SERVICE_PORT: WEB_SERVICE_PORT} if args.predictable else WEB_SERVICE_PORT, {WEB_SERVICE_TLS_PORT: WEB_SERVICE_TLS_PORT} if args.predictable else WEB_SERVICE_TLS_PORT, {BROKER_SERVICE_PORT: BROKER_SERVICE_PORT} if args.predictable else BROKER_SERVICE_PORT, {BROKER_SERVICE_TLS_PORT: BROKER_SERVICE_TLS_PORT} if args.predictable else BROKER_SERVICE_TLS_PORT] clusterdock_config_host_dir = os.path.realpath(os.path.expanduser(args.clusterdock_config_directory)) volumes = [{clusterdock_config_host_dir: CLUSTERDOCK_CLIENT_CONTAINER_DIR}] proxy_node = Node(hostname=args.proxy_node_name, group='proxy', image=node_image, ports=ports, volumes=volumes) broker_nodes = [Node(hostname=hostname, group='broker', image=node_image, volumes=volumes) for hostname in args.broker_nodes] zk_nodes = [Node(hostname=hostname, group='zookeeper', image=node_image, volumes=volumes) for hostname in args.zookeeper_nodes] nodes = [proxy_node] + broker_nodes + zk_nodes cluster = Cluster(*nodes) cluster.start(args.network) logger.info('Starting pulsar cluster (%s) version %s ...', args.pulsar_cluster_name, args.pulsar_version) # zookeeper for idx, node in enumerate(zk_nodes, start=1): zookeeper_conf = node.get_file(ZOOKEEPER_CONF) zookeeper_properties = PropertiesFile.loads(zookeeper_conf) for srvidx, srvnode in enumerate(zk_nodes, start=1): zookeeper_properties['server.{}'.format(srvidx)] = '{}.{}:2888:3888'.format(srvnode.hostname, cluster.network) node.put_file(ZOOKEEPER_CONF, PropertiesFile.dumps(zookeeper_properties)) zookeeper_commands = [ 'mkdir -p {}/data/zookeeper'.format(PULSAR_HOME), 'echo {} > {}/data/zookeeper/myid'.format(idx, PULSAR_HOME), '{}/bin/pulsar-daemon start zookeeper'.format(PULSAR_HOME) ] execute_node_command(node, ' && '.join(zookeeper_commands), quiet, 'Zookeeper start failed') web_service_url = 'http://{}.{}:{}'.format(proxy_node.hostname, cluster.network, WEB_SERVICE_PORT) web_service_url_tls = 'https://{}.{}:{}'.format(proxy_node.hostname, cluster.network, WEB_SERVICE_TLS_PORT) broker_service_url = 'pulsar://{}.{}:{}'.format(proxy_node.hostname, cluster.network, BROKER_SERVICE_PORT) broker_service_url_tls = 'pulsar+ssl://{}.{}:{}'.format(proxy_node.hostname, cluster.network, BROKER_SERVICE_TLS_PORT) init_cluster_cmd = ('{home}/bin/pulsar initialize-cluster-metadata' ' --cluster {cluster_name}' ' --zookeeper {zkhostname}.{network}:2181' ' --configuration-store {zkhostname}.{network}:2181' ' --web-service-url {web_service_url}' ' --web-service-url-tls {web_service_url_tls}' ' --broker-service-url {broker_service_url}' ' --broker-service-url-tls {broker_service_url_tls}' .format(home=PULSAR_HOME, cluster_name=args.pulsar_cluster_name, zkhostname=zk_nodes[0].hostname, hostname=proxy_node.hostname, network=cluster.network, web_service_url=web_service_url, web_service_url_tls=web_service_url_tls, broker_service_url=broker_service_url, broker_service_url_tls=broker_service_url_tls)) execute_node_command(zk_nodes[0], init_cluster_cmd, quiet, 'Cluster initialization failed') zk_servers_conf = ','.join(['{}.{}:2181'.format(node.hostname, cluster.network) for node in zk_nodes]) # bookkeepers for node in broker_nodes: bookkeeper_conf = node.get_file(BOOKKEEPER_CONF) bookkeeper_properties = PropertiesFile.loads(bookkeeper_conf) bookkeeper_properties['zkServers'] = zk_servers_conf node.put_file(BOOKKEEPER_CONF, PropertiesFile.dumps(bookkeeper_properties)) execute_node_command(node, '{}/bin/pulsar-daemon start bookie'.format(PULSAR_HOME), quiet, 'Bookkeeper start failed') execute_node_command(node, '{}/bin/bookkeeper shell bookiesanity'.format(PULSAR_HOME), quiet, 'Book keeper sanity check failed') # brokers for node in broker_nodes: broker_conf = node.get_file(BROKER_CONF) broker_properties = PropertiesFile.loads(broker_conf) broker_properties.update({'zookeeperServers': zk_servers_conf, 'configurationStoreServers': zk_servers_conf, 'clusterName': args.pulsar_cluster_name}) node.put_file(BROKER_CONF, PropertiesFile.dumps(broker_properties)) # proxy proxy_conf = proxy_node.get_file(PROXY_CONF) proxy_properties = PropertiesFile.loads(proxy_conf) proxy_properties.update({'zookeeperServers': zk_servers_conf, 'configurationStoreServers': zk_servers_conf, 'httpNumThreads': '8'}) proxy_node.put_file(PROXY_CONF, PropertiesFile.dumps(proxy_properties)) # TLS execute_node_command(proxy_node, 'rm -rf {}'.format(TLS_DIR), quiet=quiet) if args.tls: setup_commands = [ 'mkdir -p {}'.format(TLS_CLIENT_DIR), 'wget -P {} {}'.format(TLS_DIR, TLS_CONF_URL), 'mkdir -p {dir}/certs {dir}/crl {dir}/newcerts {dir}/private'.format(dir=TLS_DIR), 'chmod 700 {}/private'.format(TLS_DIR), 'touch {}/index.txt'.format(TLS_DIR), 'echo "unique_subject = no" > {}/index.txt.attr'.format(TLS_DIR), 'echo 1000 > {}/serial'.format(TLS_DIR), ] execute_node_command(proxy_node, ' && '.join(setup_commands), quiet, 'TLS system setup failed') ca_auth_commands = [ 'export CA_HOME={}'.format(TLS_DIR), 'openssl genrsa -out {dir}/private/ca.key.pem 4096'.format(dir=TLS_DIR), 'chmod 400 {}/private/ca.key.pem'.format(TLS_DIR), ('openssl req -config {dir}/openssl.cnf -key {dir}/private/ca.key.pem' ' -new -x509 -days 7300 -sha256 -extensions v3_ca -out {dir}/certs/ca.cert.pem' ' -subj "/C=US/ST=California/L=Palo Alto/O=My company/CN=*"').format(dir=TLS_DIR), 'chmod 444 {}/certs/ca.cert.pem'.format(TLS_DIR), 'cp {}/certs/ca.cert.pem {}'.format(TLS_DIR, TLS_CLIENT_DIR) ] execute_node_command(proxy_node, ' && '.join(ca_auth_commands), quiet, 'Certificate authority creation failed') server_cert_commands = [ 'export CA_HOME={}'.format(TLS_DIR), 'openssl genrsa -out {}/broker.key.pem 2048'.format(TLS_DIR), ('openssl pkcs8 -topk8 -inform PEM -outform PEM -in {dir}/broker.key.pem' ' -out {dir}/broker.key-pk8.pem -nocrypt').format(dir=TLS_DIR), # comman name (CN) needs to be *.<nw> so as that <nw> hosts can access Pulsar cluster ('openssl req -config {dir}/openssl.cnf -key {dir}/broker.key.pem -new -sha256 -out {dir}/broker.csr.pem' ' -subj "/C=US/ST=California/L=Palo Alto/O=My company/CN=*.{nw}"').format(dir=TLS_DIR, nw=cluster.network), ('openssl ca -batch -config {dir}/openssl.cnf -extensions server_cert -days 1000 -notext -md sha256' ' -in {dir}/broker.csr.pem -out {dir}/broker.cert.pem').format(dir=TLS_DIR) ] execute_node_command(proxy_node, ' && '.join(server_cert_commands), quiet, 'Broker certificate creation failed') for node in broker_nodes: broker_conf = node.get_file(BROKER_CONF) broker_properties = PropertiesFile.loads(broker_conf) broker_properties.update({'brokerServicePortTls': '6651', 'tlsEnabled': 'true', 'tlsCertificateFilePath': '{}/broker.cert.pem'.format(TLS_DIR), 'tlsKeyFilePath': '{}/broker.key-pk8.pem'.format(TLS_DIR), 'tlsTrustCertsFilePath': '{}/certs/ca.cert.pem'.format(TLS_DIR), 'webServicePortTls': '8443'}) node.put_file(BROKER_CONF, PropertiesFile.dumps(broker_properties)) proxy_conf = proxy_node.get_file(PROXY_CONF) proxy_properties = PropertiesFile.loads(proxy_conf) proxy_properties.update({'servicePortTls': '6651', 'tlsEnabledInProxy': 'true', 'tlsCertificateFilePath': '{}/broker.cert.pem'.format(TLS_DIR), 'tlsKeyFilePath': '{}/broker.key-pk8.pem'.format(TLS_DIR), 'tlsTrustCertsFilePath': '{}/certs/ca.cert.pem'.format(TLS_DIR), 'tlsEnabledWithBroker': 'true', 'brokerClientTrustCertsFilePath': '{}/certs/ca.cert.pem'.format(TLS_DIR), 'webServicePortTls': '8443'}) proxy_node.put_file(PROXY_CONF, PropertiesFile.dumps(proxy_properties)) for node in nodes: client_conf = node.get_file(CLIENT_CONF) client_properties = PropertiesFile.loads(client_conf) client_properties.update({'webServiceUrl': web_service_url_tls, 'brokerServiceUrl': broker_service_url_tls, 'useTls': 'true', 'tlsAllowInsecureConnection': 'false', 'tlsTrustCertsFilePath': '{}/certs/ca.cert.pem'.format(TLS_DIR)}) node.put_file(CLIENT_CONF, PropertiesFile.dumps(client_properties)) # TLS auth if args.tls == 'authentication': client_cert_commands = [ 'export CA_HOME={}'.format(TLS_DIR), 'openssl genrsa -out {}/admin.key.pem 2048'.format(TLS_DIR), ('openssl pkcs8 -topk8 -inform PEM -outform PEM -in {dir}/admin.key.pem' ' -out {dir}/admin.key-pk8.pem -nocrypt').format(dir=TLS_DIR), # comman name (CN) needs to be admin - same as user principal in Pulsar ('openssl req -config {dir}/openssl.cnf -key {dir}/admin.key.pem -new -sha256 -out {dir}/admin.csr.pem' ' -subj "/C=US/ST=California/L=Palo Alto/O=My company/CN=admin"').format(dir=TLS_DIR), ('openssl ca -batch -config {dir}/openssl.cnf -extensions usr_cert -days 1000 -notext -md sha256' ' -in {dir}/admin.csr.pem -out {dir}/admin.cert.pem').format(dir=TLS_DIR), 'mv {}/admin.* {}'.format(TLS_DIR, TLS_CLIENT_DIR) ] execute_node_command(proxy_node, ' && '.join(client_cert_commands), quiet, 'Client certificate creation failed') proxy_cert_commands = [ 'export CA_HOME={}'.format(TLS_DIR), 'openssl genrsa -out {}/proxy.key.pem 2048'.format(TLS_DIR), ('openssl pkcs8 -topk8 -inform PEM -outform PEM -in {dir}/proxy.key.pem' ' -out {dir}/proxy.key-pk8.pem -nocrypt').format(dir=TLS_DIR), # comman name (CN) needs to be proxyadmin - same as proxy principal in Pulsar ('openssl req -config {dir}/openssl.cnf -key {dir}/proxy.key.pem -new -sha256 -out {dir}/proxy.csr.pem' ' -subj "/C=US/ST=California/L=Palo Alto/O=My company/CN=proxyadmin"').format(dir=TLS_DIR), ('openssl ca -batch -config {dir}/openssl.cnf -extensions usr_cert -days 1000 -notext -md sha256' ' -in {dir}/proxy.csr.pem -out {dir}/proxy.cert.pem').format(dir=TLS_DIR) ] execute_node_command(proxy_node, ' && '.join(proxy_cert_commands), quiet, 'Proxy certificate creation failed') for node in broker_nodes: broker_conf = node.get_file(BROKER_CONF) broker_properties = PropertiesFile.loads(broker_conf) broker_properties.update({ 'authenticationEnabled': 'true', 'authenticationProviders': 'org.apache.pulsar.broker.authentication.AuthenticationProviderTls', 'proxyRoles': 'proxyadmin', 'superUserRoles': 'proxyadmin,admin'}) node.put_file(BROKER_CONF, PropertiesFile.dumps(broker_properties)) proxy_conf = proxy_node.get_file(PROXY_CONF) proxy_properties = PropertiesFile.loads(proxy_conf) proxy_properties.update({ 'authenticationEnabled': 'true', 'authenticationProviders': 'org.apache.pulsar.broker.authentication.AuthenticationProviderTls', 'brokerClientAuthenticationPlugin': 'org.apache.pulsar.client.impl.auth.AuthenticationTls', 'brokerClientAuthenticationParameters': ('tlsCertFile:{dir}/proxy.cert.pem,' 'tlsKeyFile:{dir}/proxy.key-pk8.pem').format(dir=TLS_DIR), 'superUserRoles': 'admin'}) proxy_node.put_file(PROXY_CONF, PropertiesFile.dumps(proxy_properties)) for node in nodes: client_conf = node.get_file(CLIENT_CONF) client_properties = PropertiesFile.loads(client_conf) client_properties.update({'authPlugin': 'org.apache.pulsar.client.impl.auth.AuthenticationTls', 'authParams': ('tlsCertFile:{dir}/admin.cert.pem,tlsKeyFile:' '{dir}/admin.key-pk8.pem').format(dir=TLS_CLIENT_DIR)}) node.put_file(CLIENT_CONF, PropertiesFile.dumps(client_properties)) # start broker nodes and proxy node for node in broker_nodes: execute_node_command(node, '{}/bin/pulsar-daemon start broker'.format(PULSAR_HOME), quiet, 'Broker start failed') out_file = '{}/logs/pulsar-proxy-{}.{}.out'.format(PULSAR_HOME, proxy_node.hostname, cluster.network) execute_node_command(proxy_node, 'mkdir -p {}/logs'.format(PULSAR_HOME), quiet) execute_node_command(proxy_node, 'nohup {}/bin/pulsar proxy > "{}" 2>&1 < /dev/null &'.format(PULSAR_HOME, out_file), quiet, 'Proxy start failed') logger.info('Performing health check on Pulsar cluster (%s) ...', args.pulsar_cluster_name) def condition(node, cluster_name, command): command_status = node.execute(command, quiet=True) return command_status.exit_code == 0 and command_status.output.splitlines()[-1].strip().strip('"') == cluster_name wait_for_condition(condition=condition, condition_args=[proxy_node, args.pulsar_cluster_name, '{}/bin/pulsar-admin clusters list'.format(PULSAR_HOME)]) logger.info('Pulsar cluster (%s) can be reached on docker network (%s):\n%s \n%s', args.pulsar_cluster_name, cluster.network, textwrap.indent('Web service URL: {}'.format(web_service_url), prefix=' '), textwrap.indent('Broker service URL: {}'.format(broker_service_url), prefix=' ')) logger.log(logging.INFO if args.tls else -1, 'Pulsar cluster (%s) can be reached securely on docker network (%s):\n%s \n%s', args.pulsar_cluster_name, cluster.network, textwrap.indent('Secure web service URL: {}'.format(web_service_url_tls), prefix=' '), textwrap.indent('Secure broker service URL: {}'.format(broker_service_url_tls), prefix=' '))
def test_propfile_preserve_trailing_comment_escape_nl(): pf = PropertiesFile.loads('#key = value\\\n') pf._check() assert dict(pf) == {} assert pf.dumps() == '#key = value\\\n'
def test_propfile_preserve_trailing_escape_nl(): pf = PropertiesFile.loads('key = value\\\n') pf._check() assert dict(pf) == {"key": "value"} assert pf.dumps() == 'key = value\\\n'
def test_propfile_preserve_comment_no_trailing_newline(): pf = PropertiesFile.loads('#key = value') pf._check() assert dict(pf) == {} assert pf.dumps() == '#key = value'
def test_propfile_dumps(): pf = PropertiesFile.loads(INPUT) pf._check() assert pf.dumps() == INPUT
def test_propfile_preserve_no_trailing_newline(): pf = PropertiesFile.loads('key = value') pf._check() assert dict(pf) == {"key": "value"} assert pf.dumps() == 'key = value'
def test_propfile_neq_string(): pf = PropertiesFile.loads('key = value\nkey: other value\n') assert pf != 'key = value\nkey: other value\n' assert 'key = value\nkey: other value\n' != pf
def test_propfile_eq_repeated_keys(): pf = PropertiesFile.loads('key = value\nkey: other value\n') pf2 = PropertiesFile.loads('key: whatever\nkey other value') assert pf == pf2 assert dict(pf) == dict(pf2) == {"key": "other value"}
def test_propfile_neq_different_comments(): pf = PropertiesFile.loads('#This is a comment.\nkey=value\n') pf2 = PropertiesFile.loads('#This is also a comment.\nkey=value\n') assert pf != pf2 assert dict(pf) == dict(pf2)