def download_and_verify(version, hash, files, base_url, plugins=DEFAULT_PLUGINS, verbose=False): print('Downloading and verifying release %s from %s' % (version, base_url)) tmp_dir = tempfile.mkdtemp() try: downloaded_files = [] print(' ' + '*' * 80) for file in files: name = os.path.basename(file) print(' Smoketest file: %s' % name) url = '%s/%s' % (base_url, file) print(' Downloading %s' % (url)) artifact_path = os.path.join(tmp_dir, file) downloaded_files.append(artifact_path) current_artifact_dir = os.path.dirname(artifact_path) os.makedirs(current_artifact_dir) urllib.request.urlretrieve(url, os.path.join(tmp_dir, file)) sha1_url = ''.join([url, '.sha1']) checksum_file = artifact_path + ".sha1" print(' Downloading %s' % (sha1_url)) urllib.request.urlretrieve(sha1_url, checksum_file) print(' Verifying checksum %s' % (checksum_file)) expected = read_fully(checksum_file) actual = sha1(artifact_path) if expected != actual: raise RuntimeError('sha1 hash for %s doesn\'t match %s != %s' % (name, expected, actual)) gpg_url = ''.join([url, '.asc']) gpg_file = artifact_path + ".asc" print(' Downloading %s' % (gpg_url)) urllib.request.urlretrieve(gpg_url, gpg_file) print(' Verifying gpg signature %s' % (gpg_file)) # here we create a temp gpg home where we download the release key as the only key into # when we verify the signature it will fail if the signed key is not in the keystore and that # way we keep the executing host unmodified since we don't have to import the key into the default keystore gpg_home_dir = os.path.join(current_artifact_dir, "gpg_home_dir") os.makedirs(gpg_home_dir, 0o700) run('gpg --homedir %s --keyserver pgp.mit.edu --recv-key D88E42B4' % gpg_home_dir, verbose=verbose) run('cd %s && gpg --homedir %s --verify %s' % (current_artifact_dir, gpg_home_dir, os.path.basename(gpg_file)), verbose=verbose) print(' ' + '*' * 80) print() smoke_test_release(version, downloaded_files, hash, plugins, verbose=verbose) print(' SUCCESS') finally: shutil.rmtree(tmp_dir)
def download_and_verify(version, hash, files, base_url, plugins=DEFAULT_PLUGINS, verbose=False): print("Downloading and verifying release %s from %s" % (version, base_url)) tmp_dir = tempfile.mkdtemp() try: downloaded_files = [] print(" " + "*" * 80) for file in files: name = os.path.basename(file) print(" Smoketest file: %s" % name) url = "%s/%s" % (base_url, file) print(" Downloading %s" % (url)) artifact_path = os.path.join(tmp_dir, file) downloaded_files.append(artifact_path) current_artifact_dir = os.path.dirname(artifact_path) os.makedirs(current_artifact_dir) urllib.request.urlretrieve(url, os.path.join(tmp_dir, file)) sha1_url = "".join([url, ".sha1"]) checksum_file = artifact_path + ".sha1" print(" Downloading %s" % (sha1_url)) urllib.request.urlretrieve(sha1_url, checksum_file) print(" Verifying checksum %s" % (checksum_file)) expected = read_fully(checksum_file) actual = sha1(artifact_path) if expected != actual: raise RuntimeError("sha1 hash for %s doesn't match %s != %s" % (name, expected, actual)) gpg_url = "".join([url, ".asc"]) gpg_file = artifact_path + ".asc" print(" Downloading %s" % (gpg_url)) urllib.request.urlretrieve(gpg_url, gpg_file) print(" Verifying gpg signature %s" % (gpg_file)) # here we create a temp gpg home where we download the release key as the only key into # when we verify the signature it will fail if the signed key is not in the keystore and that # way we keep the executing host unmodified since we don't have to import the key into the default keystore gpg_home_dir = os.path.join(current_artifact_dir, "gpg_home_dir") os.makedirs(gpg_home_dir, 0o700) run("gpg --homedir %s --keyserver pgp.mit.edu --recv-key D88E42B4" % gpg_home_dir, verbose=verbose) run( "cd %s && gpg --homedir %s --verify %s" % (current_artifact_dir, gpg_home_dir, os.path.basename(gpg_file)), verbose=verbose, ) print(" " + "*" * 80) print() smoke_test_release(version, downloaded_files, hash, plugins, verbose=verbose) print(" SUCCESS") finally: shutil.rmtree(tmp_dir)
def smoke_test_release(release, files, expected_hash, plugins): for release_file in files: if not os.path.isfile(release_file): raise RuntimeError('Smoketest failed missing file %s' % (release_file)) tmp_dir = tempfile.mkdtemp() if release_file.endswith('tar.gz'): run('tar -xzf %s -C %s' % (release_file, tmp_dir)) elif release_file.endswith('zip'): run('unzip %s -d %s' % (release_file, tmp_dir)) else: print(' Skip SmokeTest for [%s]' % release_file) continue # nothing to do here es_run_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/elasticsearch') print(' Smoke testing package [%s]' % release_file) es_plugin_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/plugin') plugin_names = {} for plugin in plugins: print(' Install plugin [%s]' % (plugin)) run('%s; %s -Des.plugins.staging=true %s %s' % (java_exe(), es_plugin_path, 'install', plugin)) plugin_names[plugin] = True if 'shield' in plugin_names: headers = { 'Authorization': 'Basic %s' % base64.b64encode(b"es_admin:foobar").decode("UTF-8") } es_shield_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/shield/esusers') print(" Install dummy shield user") run('%s; %s useradd es_admin -r admin -p foobar' % (java_exe(), es_shield_path)) else: headers = {} print(' Starting elasticsearch deamon from [%s]' % os.path.join(tmp_dir, 'elasticsearch-%s' % release)) try: run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.script.inline=true -Des.script.indexed=true -Des.repositories.url.allowed_urls=http://snapshot.test* %s -Des.pidfile=%s' % (java_exe(), es_run_path, '-d', os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'es-smoke.pid'))) conn = HTTPConnection(host='127.0.0.1', port=9200, timeout=20) if not wait_for_node_startup(header=headers): print("elasticsearch logs:") print('*' * 80) logs = read_fully( os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'logs/prepare_release.log')) print(logs) print('*' * 80) raise RuntimeError('server didn\'t start up') try: # we now get / and /_nodes to fetch basic infos like hashes etc and the installed plugins conn.request('GET', '', headers=headers) res = conn.getresponse() if res.status == 200: version = json.loads(res.read().decode("utf-8"))['version'] if release != version['number']: raise RuntimeError( 'Expected version [%s] but was [%s]' % (release, version['number'])) if version['build_snapshot']: raise RuntimeError('Expected non snapshot version') if expected_hash.startswith(version['build_hash'].strip()): raise RuntimeError( 'HEAD hash does not match expected [%s] but got [%s]' % (expected_hash, version['build_hash'])) print(' Verify if plugins are listed in _nodes') conn.request('GET', '/_nodes?plugin=true&pretty=true', headers=headers) res = conn.getresponse() if res.status == 200: nodes = json.loads(res.read().decode("utf-8"))['nodes'] for _, node in nodes.items(): node_plugins = node['plugins'] for node_plugin in node_plugins: if not plugin_names.get( node_plugin['name'].strip(), False): raise RuntimeError('Unexpeced plugin %s' % node_plugin['name']) del plugin_names[node_plugin['name']] if plugin_names: raise RuntimeError('Plugins not loaded %s' % list(plugin_names.keys())) else: raise RuntimeError('Expected HTTP 200 but got %s' % res.status) else: raise RuntimeError('Expected HTTP 200 but got %s' % res.status) finally: conn.close() finally: pid_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'es-smoke.pid') if os.path.exists( pid_path): # try reading the pid and kill the node pid = int(read_fully(pid_path)) os.kill(pid, signal.SIGKILL) shutil.rmtree(tmp_dir) print(' ' + '*' * 80) print()
def smoke_test_release(release, files, expected_hash, plugins): for release_file in files: if not os.path.isfile(release_file): raise RuntimeError('Smoketest failed missing file %s' % (release_file)) tmp_dir = tempfile.mkdtemp() if release_file.endswith('tar.gz'): run('tar -xzf %s -C %s' % (release_file, tmp_dir)) elif release_file.endswith('zip'): run('unzip %s -d %s' % (release_file, tmp_dir)) else: print(' Skip SmokeTest for [%s]' % release_file) continue # nothing to do here es_run_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/elasticsearch') print(' Smoke testing package [%s]' % release_file) es_plugin_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/plugin') plugin_names = {} for plugin in plugins: print(' Install plugin [%s]' % (plugin)) run('%s; %s -Des.plugins.staging=true %s %s' % (java_exe(), es_plugin_path, 'install', plugin)) plugin_names[plugin] = True if 'shield' in plugin_names: headers = { 'Authorization' : 'Basic %s' % base64.b64encode(b"es_admin:foobar").decode("UTF-8") } es_shield_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/shield/esusers') print(" Install dummy shield user") run('%s; %s useradd es_admin -r admin -p foobar' % (java_exe(), es_shield_path)) else: headers = {} print(' Starting elasticsearch deamon from [%s]' % os.path.join(tmp_dir, 'elasticsearch-%s' % release)) try: run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.script.inline=on -Des.script.indexed=on -Des.repositories.url.allowed_urls=http://snapshot.test* %s -Des.pidfile=%s' % (java_exe(), es_run_path, '-d', os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'es-smoke.pid'))) conn = HTTPConnection(host='127.0.0.1', port=9200, timeout=20) if not wait_for_node_startup(header=headers): print("elasticsearch logs:") print('*' * 80) logs = read_fully(os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'logs/prepare_release.log')) print(logs) print('*' * 80) raise RuntimeError('server didn\'t start up') try: # we now get / and /_nodes to fetch basic infos like hashes etc and the installed plugins conn.request('GET', '', headers=headers) res = conn.getresponse() if res.status == 200: version = json.loads(res.read().decode("utf-8"))['version'] if release != version['number']: raise RuntimeError('Expected version [%s] but was [%s]' % (release, version['number'])) if version['build_snapshot']: raise RuntimeError('Expected non snapshot version') if expected_hash.startswith(version['build_hash'].strip()): raise RuntimeError('HEAD hash does not match expected [%s] but got [%s]' % (expected_hash, version['build_hash'])) print(' Verify if plugins are listed in _nodes') conn.request('GET', '/_nodes?plugin=true&pretty=true', headers=headers) res = conn.getresponse() if res.status == 200: nodes = json.loads(res.read().decode("utf-8"))['nodes'] for _, node in nodes.items(): node_plugins = node['plugins'] for node_plugin in node_plugins: if not plugin_names.get(node_plugin['name'].strip(), False): raise RuntimeError('Unexpeced plugin %s' % node_plugin['name']) del plugin_names[node_plugin['name']] if plugin_names: raise RuntimeError('Plugins not loaded %s' % list(plugin_names.keys())) else: raise RuntimeError('Expected HTTP 200 but got %s' % res.status) else: raise RuntimeError('Expected HTTP 200 but got %s' % res.status) finally: conn.close() finally: pid_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'es-smoke.pid') if os.path.exists(pid_path): # try reading the pid and kill the node pid = int(read_fully(pid_path)) os.kill(pid, signal.SIGKILL) shutil.rmtree(tmp_dir) print(' ' + '*' * 80) print()
def smoke_test_release(release, files, expected_hash, plugins): for release_file in files: if not os.path.isfile(release_file): raise RuntimeError("Smoketest failed missing file %s" % (release_file)) tmp_dir = tempfile.mkdtemp() if release_file.endswith("tar.gz"): run("tar -xzf %s -C %s" % (release_file, tmp_dir)) elif release_file.endswith("zip"): run("unzip %s -d %s" % (release_file, tmp_dir)) else: print(" Skip SmokeTest for [%s]" % release_file) continue # nothing to do here es_dir = os.path.join(tmp_dir, "elasticsearch-%s" % (release)) es_run_path = os.path.join(es_dir, "bin/elasticsearch") print(" Smoke testing package [%s]" % release_file) es_plugin_path = os.path.join(es_dir, "bin/elasticsearch-plugin") plugin_names = {} for plugin in plugins: print(" Install plugin [%s]" % (plugin)) run( '%s; export ES_JAVA_OPTS="-Des.plugins.staging=%s"; %s %s %s' % (java_exe(), expected_hash, es_plugin_path, "install -b", plugin) ) plugin_names[plugin] = True if "x-pack" in plugin_names: headers = {"Authorization": "Basic %s" % base64.b64encode(b"es_admin:foobar").decode("UTF-8")} es_shield_path = os.path.join(es_dir, "bin/x-pack/users") print(" Install dummy shield user") run("%s; %s useradd es_admin -r superuser -p foobar" % (java_exe(), es_shield_path)) else: headers = {} print(" Starting elasticsearch deamon from [%s]" % es_dir) try: run( "%s; %s -Enode.name=smoke_tester -Ecluster.name=prepare_release -Escript.inline=true -Escript.stored=true -Erepositories.url.allowed_urls=http://snapshot.test* %s -Epidfile=%s -Enode.portsfile=true" % (java_exe(), es_run_path, "-d", os.path.join(es_dir, "es-smoke.pid")) ) if not wait_for_node_startup(es_dir, header=headers): print("elasticsearch logs:") print("*" * 80) logs = read_fully(os.path.join(es_dir, "logs/prepare_release.log")) print(logs) print("*" * 80) raise RuntimeError("server didn't start up") try: # we now get / and /_nodes to fetch basic infos like hashes etc and the installed plugins host = get_host_from_ports_file(es_dir) conn = HTTPConnection(host, timeout=20) conn.request("GET", "/", headers=headers) res = conn.getresponse() if res.status == 200: version = json.loads(res.read().decode("utf-8"))["version"] if release != version["number"]: raise RuntimeError("Expected version [%s] but was [%s]" % (release, version["number"])) if version["build_snapshot"]: raise RuntimeError("Expected non snapshot version") if expected_hash != version["build_hash"].strip(): raise RuntimeError( "HEAD hash does not match expected [%s] but got [%s]" % (expected_hash, version["build_hash"]) ) print(" Verify if plugins are listed in _nodes") conn.request("GET", "/_nodes?plugin=true&pretty=true", headers=headers) res = conn.getresponse() if res.status == 200: nodes = json.loads(res.read().decode("utf-8"))["nodes"] for _, node in nodes.items(): node_plugins = node["plugins"] for node_plugin in node_plugins: if not plugin_names.get(node_plugin["name"].strip(), False): raise RuntimeError("Unexpected plugin %s" % node_plugin["name"]) del plugin_names[node_plugin["name"]] if plugin_names: raise RuntimeError("Plugins not loaded %s" % list(plugin_names.keys())) else: raise RuntimeError("Expected HTTP 200 but got %s" % res.status) else: raise RuntimeError("Expected HTTP 200 but got %s" % res.status) finally: conn.close() finally: pid_path = os.path.join(es_dir, "es-smoke.pid") if os.path.exists(pid_path): # try reading the pid and kill the node pid = int(read_fully(pid_path)) os.kill(pid, signal.SIGKILL) shutil.rmtree(tmp_dir) print(" " + "*" * 80) print()