def download_and_verify(version,
hash,
files,
base_url,
plugins=DEFAULT_PLUGINS,
verbose=False):
print('Downloading and verifying release %s from %s' % (version, base_url))
tmp_dir = tempfile.mkdtemp()
try:
downloaded_files = []
print(' ' + '*' * 80)
for file in files:
name = os.path.basename(file)
print(' Smoketest file: %s' % name)
url = '%s/%s' % (base_url, file)
print(' Downloading %s' % (url))
artifact_path = os.path.join(tmp_dir, file)
downloaded_files.append(artifact_path)
current_artifact_dir = os.path.dirname(artifact_path)
os.makedirs(current_artifact_dir)
urllib.request.urlretrieve(url, os.path.join(tmp_dir, file))
sha1_url = ''.join([url, '.sha1'])
checksum_file = artifact_path + ".sha1"
print(' Downloading %s' % (sha1_url))
urllib.request.urlretrieve(sha1_url, checksum_file)
print(' Verifying checksum %s' % (checksum_file))
expected = read_fully(checksum_file)
actual = sha1(artifact_path)
if expected != actual:
raise RuntimeError('sha1 hash for %s doesn\'t match %s != %s' %
(name, expected, actual))
gpg_url = ''.join([url, '.asc'])
gpg_file = artifact_path + ".asc"
print(' Downloading %s' % (gpg_url))
urllib.request.urlretrieve(gpg_url, gpg_file)
print(' Verifying gpg signature %s' % (gpg_file))
# here we create a temp gpg home where we download the release key as the only key into
# when we verify the signature it will fail if the signed key is not in the keystore and that
# way we keep the executing host unmodified since we don't have to import the key into the default keystore
gpg_home_dir = os.path.join(current_artifact_dir, "gpg_home_dir")
os.makedirs(gpg_home_dir, 0o700)
run('gpg --homedir %s --keyserver pgp.mit.edu --recv-key D88E42B4'
% gpg_home_dir,
verbose=verbose)
run('cd %s && gpg --homedir %s --verify %s' %
(current_artifact_dir, gpg_home_dir,
os.path.basename(gpg_file)),
verbose=verbose)
print(' ' + '*' * 80)
print()
smoke_test_release(version,
downloaded_files,
hash,
plugins,
verbose=verbose)
print(' SUCCESS')
finally:
shutil.rmtree(tmp_dir)
def download_and_verify(version, hash, files, base_url, plugins=DEFAULT_PLUGINS, verbose=False):
print("Downloading and verifying release %s from %s" % (version, base_url))
tmp_dir = tempfile.mkdtemp()
try:
downloaded_files = []
print(" " + "*" * 80)
for file in files:
name = os.path.basename(file)
print(" Smoketest file: %s" % name)
url = "%s/%s" % (base_url, file)
print(" Downloading %s" % (url))
artifact_path = os.path.join(tmp_dir, file)
downloaded_files.append(artifact_path)
current_artifact_dir = os.path.dirname(artifact_path)
os.makedirs(current_artifact_dir)
urllib.request.urlretrieve(url, os.path.join(tmp_dir, file))
sha1_url = "".join([url, ".sha1"])
checksum_file = artifact_path + ".sha1"
print(" Downloading %s" % (sha1_url))
urllib.request.urlretrieve(sha1_url, checksum_file)
print(" Verifying checksum %s" % (checksum_file))
expected = read_fully(checksum_file)
actual = sha1(artifact_path)
if expected != actual:
raise RuntimeError("sha1 hash for %s doesn't match %s != %s" % (name, expected, actual))
gpg_url = "".join([url, ".asc"])
gpg_file = artifact_path + ".asc"
print(" Downloading %s" % (gpg_url))
urllib.request.urlretrieve(gpg_url, gpg_file)
print(" Verifying gpg signature %s" % (gpg_file))
# here we create a temp gpg home where we download the release key as the only key into
# when we verify the signature it will fail if the signed key is not in the keystore and that
# way we keep the executing host unmodified since we don't have to import the key into the default keystore
gpg_home_dir = os.path.join(current_artifact_dir, "gpg_home_dir")
os.makedirs(gpg_home_dir, 0o700)
run("gpg --homedir %s --keyserver pgp.mit.edu --recv-key D88E42B4" % gpg_home_dir, verbose=verbose)
run(
"cd %s && gpg --homedir %s --verify %s"
% (current_artifact_dir, gpg_home_dir, os.path.basename(gpg_file)),
verbose=verbose,
)
print(" " + "*" * 80)
print()
smoke_test_release(version, downloaded_files, hash, plugins, verbose=verbose)
print(" SUCCESS")
finally:
shutil.rmtree(tmp_dir)
def smoke_test_release(release, files, expected_hash, plugins):
for release_file in files:
if not os.path.isfile(release_file):
raise RuntimeError('Smoketest failed missing file %s' %
(release_file))
tmp_dir = tempfile.mkdtemp()
if release_file.endswith('tar.gz'):
run('tar -xzf %s -C %s' % (release_file, tmp_dir))
elif release_file.endswith('zip'):
run('unzip %s -d %s' % (release_file, tmp_dir))
else:
print(' Skip SmokeTest for [%s]' % release_file)
continue # nothing to do here
es_run_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release),
'bin/elasticsearch')
print(' Smoke testing package [%s]' % release_file)
es_plugin_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release),
'bin/plugin')
plugin_names = {}
for plugin in plugins:
print(' Install plugin [%s]' % (plugin))
run('%s; %s -Des.plugins.staging=true %s %s' %
(java_exe(), es_plugin_path, 'install', plugin))
plugin_names[plugin] = True
if 'shield' in plugin_names:
headers = {
'Authorization':
'Basic %s' %
base64.b64encode(b"es_admin:foobar").decode("UTF-8")
}
es_shield_path = os.path.join(tmp_dir,
'elasticsearch-%s' % (release),
'bin/shield/esusers')
print(" Install dummy shield user")
run('%s; %s useradd es_admin -r admin -p foobar' %
(java_exe(), es_shield_path))
else:
headers = {}
print(' Starting elasticsearch deamon from [%s]' %
os.path.join(tmp_dir, 'elasticsearch-%s' % release))
try:
run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.script.inline=true -Des.script.indexed=true -Des.repositories.url.allowed_urls=http://snapshot.test* %s -Des.pidfile=%s'
% (java_exe(), es_run_path, '-d',
os.path.join(tmp_dir, 'elasticsearch-%s' %
(release), 'es-smoke.pid')))
conn = HTTPConnection(host='127.0.0.1', port=9200, timeout=20)
if not wait_for_node_startup(header=headers):
print("elasticsearch logs:")
print('*' * 80)
logs = read_fully(
os.path.join(tmp_dir, 'elasticsearch-%s' % (release),
'logs/prepare_release.log'))
print(logs)
print('*' * 80)
raise RuntimeError('server didn\'t start up')
try: # we now get / and /_nodes to fetch basic infos like hashes etc and the installed plugins
conn.request('GET', '', headers=headers)
res = conn.getresponse()
if res.status == 200:
version = json.loads(res.read().decode("utf-8"))['version']
if release != version['number']:
raise RuntimeError(
'Expected version [%s] but was [%s]' %
(release, version['number']))
if version['build_snapshot']:
raise RuntimeError('Expected non snapshot version')
if expected_hash.startswith(version['build_hash'].strip()):
raise RuntimeError(
'HEAD hash does not match expected [%s] but got [%s]'
% (expected_hash, version['build_hash']))
print(' Verify if plugins are listed in _nodes')
conn.request('GET',
'/_nodes?plugin=true&pretty=true',
headers=headers)
res = conn.getresponse()
if res.status == 200:
nodes = json.loads(res.read().decode("utf-8"))['nodes']
for _, node in nodes.items():
node_plugins = node['plugins']
for node_plugin in node_plugins:
if not plugin_names.get(
node_plugin['name'].strip(), False):
raise RuntimeError('Unexpeced plugin %s' %
node_plugin['name'])
del plugin_names[node_plugin['name']]
if plugin_names:
raise RuntimeError('Plugins not loaded %s' %
list(plugin_names.keys()))
else:
raise RuntimeError('Expected HTTP 200 but got %s' %
res.status)
else:
raise RuntimeError('Expected HTTP 200 but got %s' %
res.status)
finally:
conn.close()
finally:
pid_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release),
'es-smoke.pid')
if os.path.exists(
pid_path): # try reading the pid and kill the node
pid = int(read_fully(pid_path))
os.kill(pid, signal.SIGKILL)
shutil.rmtree(tmp_dir)
print(' ' + '*' * 80)
print()
def smoke_test_release(release, files, expected_hash, plugins):
for release_file in files:
if not os.path.isfile(release_file):
raise RuntimeError('Smoketest failed missing file %s' % (release_file))
tmp_dir = tempfile.mkdtemp()
if release_file.endswith('tar.gz'):
run('tar -xzf %s -C %s' % (release_file, tmp_dir))
elif release_file.endswith('zip'):
run('unzip %s -d %s' % (release_file, tmp_dir))
else:
print(' Skip SmokeTest for [%s]' % release_file)
continue # nothing to do here
es_run_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/elasticsearch')
print(' Smoke testing package [%s]' % release_file)
es_plugin_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/plugin')
plugin_names = {}
for plugin in plugins:
print(' Install plugin [%s]' % (plugin))
run('%s; %s -Des.plugins.staging=true %s %s' % (java_exe(), es_plugin_path, 'install', plugin))
plugin_names[plugin] = True
if 'shield' in plugin_names:
headers = { 'Authorization' : 'Basic %s' % base64.b64encode(b"es_admin:foobar").decode("UTF-8") }
es_shield_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/shield/esusers')
print(" Install dummy shield user")
run('%s; %s useradd es_admin -r admin -p foobar' % (java_exe(), es_shield_path))
else:
headers = {}
print(' Starting elasticsearch deamon from [%s]' % os.path.join(tmp_dir, 'elasticsearch-%s' % release))
try:
run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.script.inline=on -Des.script.indexed=on -Des.repositories.url.allowed_urls=http://snapshot.test* %s -Des.pidfile=%s'
% (java_exe(), es_run_path, '-d', os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'es-smoke.pid')))
conn = HTTPConnection(host='127.0.0.1', port=9200, timeout=20)
if not wait_for_node_startup(header=headers):
print("elasticsearch logs:")
print('*' * 80)
logs = read_fully(os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'logs/prepare_release.log'))
print(logs)
print('*' * 80)
raise RuntimeError('server didn\'t start up')
try: # we now get / and /_nodes to fetch basic infos like hashes etc and the installed plugins
conn.request('GET', '', headers=headers)
res = conn.getresponse()
if res.status == 200:
version = json.loads(res.read().decode("utf-8"))['version']
if release != version['number']:
raise RuntimeError('Expected version [%s] but was [%s]' % (release, version['number']))
if version['build_snapshot']:
raise RuntimeError('Expected non snapshot version')
if expected_hash.startswith(version['build_hash'].strip()):
raise RuntimeError('HEAD hash does not match expected [%s] but got [%s]' % (expected_hash, version['build_hash']))
print(' Verify if plugins are listed in _nodes')
conn.request('GET', '/_nodes?plugin=true&pretty=true', headers=headers)
res = conn.getresponse()
if res.status == 200:
nodes = json.loads(res.read().decode("utf-8"))['nodes']
for _, node in nodes.items():
node_plugins = node['plugins']
for node_plugin in node_plugins:
if not plugin_names.get(node_plugin['name'].strip(), False):
raise RuntimeError('Unexpeced plugin %s' % node_plugin['name'])
del plugin_names[node_plugin['name']]
if plugin_names:
raise RuntimeError('Plugins not loaded %s' % list(plugin_names.keys()))
else:
raise RuntimeError('Expected HTTP 200 but got %s' % res.status)
else:
raise RuntimeError('Expected HTTP 200 but got %s' % res.status)
finally:
conn.close()
finally:
pid_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'es-smoke.pid')
if os.path.exists(pid_path): # try reading the pid and kill the node
pid = int(read_fully(pid_path))
os.kill(pid, signal.SIGKILL)
shutil.rmtree(tmp_dir)
print(' ' + '*' * 80)
print()
def smoke_test_release(release, files, expected_hash, plugins):
for release_file in files:
if not os.path.isfile(release_file):
raise RuntimeError("Smoketest failed missing file %s" % (release_file))
tmp_dir = tempfile.mkdtemp()
if release_file.endswith("tar.gz"):
run("tar -xzf %s -C %s" % (release_file, tmp_dir))
elif release_file.endswith("zip"):
run("unzip %s -d %s" % (release_file, tmp_dir))
else:
print(" Skip SmokeTest for [%s]" % release_file)
continue # nothing to do here
es_dir = os.path.join(tmp_dir, "elasticsearch-%s" % (release))
es_run_path = os.path.join(es_dir, "bin/elasticsearch")
print(" Smoke testing package [%s]" % release_file)
es_plugin_path = os.path.join(es_dir, "bin/elasticsearch-plugin")
plugin_names = {}
for plugin in plugins:
print(" Install plugin [%s]" % (plugin))
run(
'%s; export ES_JAVA_OPTS="-Des.plugins.staging=%s"; %s %s %s'
% (java_exe(), expected_hash, es_plugin_path, "install -b", plugin)
)
plugin_names[plugin] = True
if "x-pack" in plugin_names:
headers = {"Authorization": "Basic %s" % base64.b64encode(b"es_admin:foobar").decode("UTF-8")}
es_shield_path = os.path.join(es_dir, "bin/x-pack/users")
print(" Install dummy shield user")
run("%s; %s useradd es_admin -r superuser -p foobar" % (java_exe(), es_shield_path))
else:
headers = {}
print(" Starting elasticsearch deamon from [%s]" % es_dir)
try:
run(
"%s; %s -Enode.name=smoke_tester -Ecluster.name=prepare_release -Escript.inline=true -Escript.stored=true -Erepositories.url.allowed_urls=http://snapshot.test* %s -Epidfile=%s -Enode.portsfile=true"
% (java_exe(), es_run_path, "-d", os.path.join(es_dir, "es-smoke.pid"))
)
if not wait_for_node_startup(es_dir, header=headers):
print("elasticsearch logs:")
print("*" * 80)
logs = read_fully(os.path.join(es_dir, "logs/prepare_release.log"))
print(logs)
print("*" * 80)
raise RuntimeError("server didn't start up")
try: # we now get / and /_nodes to fetch basic infos like hashes etc and the installed plugins
host = get_host_from_ports_file(es_dir)
conn = HTTPConnection(host, timeout=20)
conn.request("GET", "/", headers=headers)
res = conn.getresponse()
if res.status == 200:
version = json.loads(res.read().decode("utf-8"))["version"]
if release != version["number"]:
raise RuntimeError("Expected version [%s] but was [%s]" % (release, version["number"]))
if version["build_snapshot"]:
raise RuntimeError("Expected non snapshot version")
if expected_hash != version["build_hash"].strip():
raise RuntimeError(
"HEAD hash does not match expected [%s] but got [%s]"
% (expected_hash, version["build_hash"])
)
print(" Verify if plugins are listed in _nodes")
conn.request("GET", "/_nodes?plugin=true&pretty=true", headers=headers)
res = conn.getresponse()
if res.status == 200:
nodes = json.loads(res.read().decode("utf-8"))["nodes"]
for _, node in nodes.items():
node_plugins = node["plugins"]
for node_plugin in node_plugins:
if not plugin_names.get(node_plugin["name"].strip(), False):
raise RuntimeError("Unexpected plugin %s" % node_plugin["name"])
del plugin_names[node_plugin["name"]]
if plugin_names:
raise RuntimeError("Plugins not loaded %s" % list(plugin_names.keys()))
else:
raise RuntimeError("Expected HTTP 200 but got %s" % res.status)
else:
raise RuntimeError("Expected HTTP 200 but got %s" % res.status)
finally:
conn.close()
finally:
pid_path = os.path.join(es_dir, "es-smoke.pid")
if os.path.exists(pid_path): # try reading the pid and kill the node
pid = int(read_fully(pid_path))
os.kill(pid, signal.SIGKILL)
shutil.rmtree(tmp_dir)
print(" " + "*" * 80)
print()
