def test_run_subprocess_fail(self):
if os.path.exists('/bin/false'):
false_path = '/bin/false'
elif os.path.exists('/usr/bin/false'):
false_path = '/usr/bin/false'
else:
raise NotImplementedError('Unsupported test platform.')
got, output = run_subprocess(false_path)
self.assertEquals((1, ''), (got, output))
got, output = run_subprocess('/bin/ls /abc/def')
self.assertNotEquals(0, got)
self.assertTrue(output.find('No such file or directory') >= 0)
def test_run_subprocess_fail(self):
if os.path.exists("/bin/false"):
false_path = "/bin/false"
elif os.path.exists("/usr/bin/false"):
false_path = "/usr/bin/false"
else:
raise NotImplementedError("Unsupported test platform.")
got, output = run_subprocess(false_path)
self.assertEqual((1, ""), (got, output))
got, output = run_subprocess("/bin/ls /abc/def")
self.assertNotEqual(0, got)
self.assertTrue(output.find("No such file or directory") >= 0)
def test_run_subprocess_fail(self):
if os.path.exists('/bin/false'):
false_path = '/bin/false'
elif os.path.exists('/usr/bin/false'):
false_path = '/usr/bin/false'
else:
raise NotImplementedError('Unsupported test platform.')
got, output = run_subprocess(false_path)
self.assertEqual((1, ''), (got, output))
got, output = run_subprocess('/bin/ls /abc/def')
self.assertNotEqual(0, got)
self.assertTrue(output.find('No such file or directory') >= 0)
def do_run_subprocess_ok(self, check, logfile=None):
if os.path.exists('/bin/true'):
true_path = '/bin/true'
elif os.path.exists('/usr/bin/true'):
true_path = '/usr/bin/true'
else:
raise NotImplementedError('Unsupported test platform.')
tests = [(true_path, ''), ('/bin/echo Hello', 'Hello'),
('/bin/echo "Hello"', 'Hello'),
('/bin/echo "Hello World"', 'Hello World'),
('/bin/echo "Hello\nWorld"', 'Hello\nWorld'),
('/bin/echo \'"Hello World"\'', '"Hello World"')]
for cmd, expect in tests:
if logfile:
output = check_subprocesses_to_logfile('Test Logfile', logfile,
[cmd])
elif check:
output = check_subprocess(cmd)
else:
code, output = run_subprocess(cmd)
self.assertEqual(0, code)
if logfile:
self.assertTrue(os.path.exists(logfile))
self.assertIsNone(output)
with io.open(logfile, 'r', encoding='utf-8') as stream:
lines = stream.read().split('\n')
self.assertTrue('Spawning' in lines[0])
self.assertTrue('process completed with' in lines[-2])
body = '\n'.join(lines[3:-3]).strip()
self.assertEqual(expect, body)
else:
self.assertEqual(expect, output)
def do_run_subprocess_ok(self, check, logfile=None):
if os.path.exists('/bin/true'):
true_path = '/bin/true'
elif os.path.exists('/usr/bin/true'):
true_path = '/usr/bin/true'
else:
raise NotImplementedError('Unsupported test platform.')
tests = [(true_path, ''),
('/bin/echo Hello', 'Hello'),
('/bin/echo "Hello"', 'Hello'),
('/bin/echo "Hello World"', 'Hello World'),
('/bin/echo "Hello\nWorld"', 'Hello\nWorld'),
('/bin/echo \'"Hello World"\'', '"Hello World"')]
for cmd, expect in tests:
if logfile:
output = check_subprocesses_to_logfile('Test Logfile', logfile, [cmd])
elif check:
output = check_subprocess(cmd)
else:
code, output = run_subprocess(cmd)
self.assertEqual(0, code)
if logfile:
self.assertTrue(os.path.exists(logfile))
self.assertIsNone(output)
with io.open(logfile, 'r', encoding='utf-8') as stream:
lines = stream.read().split('\n')
self.assertTrue('Spawning' in lines[0])
self.assertTrue('process completed with' in lines[-2])
body = '\n'.join(lines[3:-3]).strip()
self.assertEqual(expect, body)
else:
self.assertEqual(expect, output)
def __tag_images(versions_to_tag, project, account, project_images,
bom_contents_by_name):
images_to_tag = set([])
for bom_version in versions_to_tag:
to_tag = [
i for i in __derive_images_from_bom(
bom_version, bom_contents_by_name) if i in project_images
]
images_to_tag.update(to_tag)
for image in images_to_tag:
return_code, stdout = run_subprocess(
'gcloud compute images describe'
' --project={project} --account={account} --format=json {image}'.
format(project=project, account=account, image=image),
echo=False)
# Adding labels is idempotent, adding the same label again doesn't break anything.
if not return_code:
payload_str = stdout.strip()
timestamp = json.loads(payload_str)['creationTimestamp']
timestamp = timestamp[:timestamp.index('T')]
check_subprocess(
'gcloud compute images add-labels --project={project} --account={account} --labels={key}={timestamp} {image}'
.format(project=project,
account=account,
key=PUBLISHED_TAG_KEY,
timestamp=timestamp,
image=image))
def check_gcloud():
retcode, _ = run_subprocess('gcloud --version')
if not retcode:
return
sys.stderr.write('ERROR: This program requires gcloud. To obtain gcloud:\n'
' curl https://sdk.cloud.google.com | bash\n')
sys.exit(-1)
def check_gcloud():
retcode, _ = run_subprocess('gcloud --version')
if not retcode:
return
sys.stderr.write('ERROR: This program requires gcloud. To obtain gcloud:\n'
' curl https://sdk.cloud.google.com | bash\n')
sys.exit(-1)
def try_until_ready(command):
while True:
retcode, stdout = run_subprocess(command)
if not retcode:
break
if stdout.find('refused') > 0:
print('New instance does not seem ready yet...retry in 5s.')
else:
print(stdout.strip())
print('Retrying in 5s.')
time.sleep(5)
def test_run_subprocess_get_pid(self):
# See if we can run a job by looking up our job
# This is also testing parsing command lines.
code, output = run_subprocess('/bin/ps -x')
self.assertEquals(0, code)
my_pid = '%d ' % os.getpid()
candidates = [line for line in output.split('\n')
if line.find(my_pid) >= 0 and line.find('python') > 0]
if len(candidates) != 1:
logging.error('Unexpected output\n%s', output)
self.assertEquals(1, len(candidates))
self.assertTrue(candidates[0].find(' python ') > 0)
def test_run_subprocess_get_pid(self):
# See if we can run a job by looking up our job
# This is also testing parsing command lines.
code, output = run_subprocess('/bin/ps -x')
self.assertEqual(0, code)
my_pid = '%d ' % os.getpid()
candidates = [line for line in output.split('\n')
if line.find(my_pid) >= 0 and line.find('python') > 0]
if len(candidates) != 1:
logging.error('Unexpected output\n%s', output)
self.assertEqual(1, len(candidates))
self.assertTrue(candidates[0].find(' python ') > 0)
def __gcb_image_exists(self, image_name, version):
"""Determine if gcb image already exists."""
options = self.options
command = [
'gcloud', 'beta', '--account', options.gcb_service_account,
'artifacts', 'docker', 'images', 'describe',
'%s/%s:%s' % (options.artifact_registry, image_name, version)
]
exit_code = run_subprocess(' '.join(command),
stderr=subprocess.PIPE)[0]
return exit_code == 0
def try_until_ready(command):
while True:
retcode, stdout = run_subprocess(command)
if not retcode:
break
if stdout.find('refused') > 0:
print('New instance does not seem ready yet...retry in 5s.')
else:
print(stdout.strip())
print('Retrying in 5s.')
time.sleep(5)
def load_halyard_version_commits(self):
logging.debug('Fetching existing halyard build versions')
retcode, stdout = run_subprocess('gsutil cat ' + self.__versions_url)
if not retcode:
contents = stdout + '\n'
else:
if stdout.find('No URLs matched') < 0:
raise_and_log_error(
ExecutionError('No URLs matched', program='gsutil'),
'Could not fetch "%s": %s' % (self.__versions_url, stdout))
contents = ''
logging.warning('%s did not exist. Creating a new one.',
self.__versions_url)
return contents
def load_halyard_version_commits(self):
logging.debug('Fetching existing halyard build versions')
retcode, stdout = run_subprocess('gsutil cat ' + self.__versions_url)
if not retcode:
contents = stdout + '\n'
else:
if stdout.find('No URLs matched') < 0:
raise_and_log_error(
ExecutionError('No URLs matched', program='gsutil'),
'Could not fetch "%s": %s' % (self.__versions_url, stdout))
contents = ''
logging.warning(
'%s did not exist. Creating a new one.', self.__versions_url)
return contents
def load_halyard_version_commits(self):
logging.debug("Fetching existing halyard build versions")
retcode, stdout = run_subprocess("gsutil cat " + self.__versions_url)
if not retcode:
contents = stdout + "\n"
else:
if stdout.find("No URLs matched") < 0:
raise_and_log_error(
ExecutionError("No URLs matched", program="gsutil"),
'Could not fetch "%s": %s' % (self.__versions_url, stdout),
)
contents = ""
logging.warning("%s did not exist. Creating a new one.",
self.__versions_url)
return contents
def __write_image_delete_script(
possible_versions_to_delete,
days_before,
project,
account,
project_images,
bom_contents_by_name,
):
images_to_delete = set([])
print("Calculating images for {} versions to delete.".format(
len(possible_versions_to_delete)))
for bom_version in possible_versions_to_delete:
deletable = [
i for i in __derive_images_from_bom(
bom_version, bom_contents_by_name) if i in project_images
]
images_to_delete.update(deletable)
delete_script_lines = []
for image in images_to_delete:
return_code, stdout = run_subprocess(
"gcloud compute images describe"
" --project={project} --account={account} --format=json {image}".
format(project=project, account=account, image=image),
echo=False,
)
json_str = ""
if return_code:
# Some BOMs may refer to service versions without HA images.
print(
"Lookup for image {image} in project {project} failed, ignoring"
.format(image=image, project=project))
continue
else:
json_str = stdout.strip()
payload = json.loads(json_str)
if __image_age_days(payload) > days_before:
labels = payload.get("labels", None)
if not labels or PUBLISHED_TAG_KEY not in labels:
line = "gcloud compute images delete --project={project} --account={account} {image} -q".format(
project=project, account=account, image=image)
delete_script_lines.append(line)
delete_script = "\n".join(delete_script_lines)
timestamp = "{:%Y%m%d%H%M%S}".format(datetime.datetime.utcnow())
script_name = "delete-images-{}".format(timestamp)
with open(script_name, "w") as script:
script.write(delete_script)
print("Wrote image janitor script to {}".format(script_name))
def build_report(test_controller):
"""Report on the test results."""
options = test_controller.options
citest_log_dir = os.path.join(options.log_dir, 'citest_logs')
if not os.path.exists(citest_log_dir):
logging.warning('%s does not exist -- no citest logs.', citest_log_dir)
return None
retcode, stdout = run_subprocess(
'python -m citest.reporting.generate_html_report --index *.journal',
shell=True,
cwd=citest_log_dir)
if retcode != 0:
logging.error('Error building report: %s', stdout)
logging.info('Logging information is in %s', options.log_dir)
return test_controller.build_summary()
def do_run_subprocess_ok(self, check):
if os.path.exists('/bin/true'):
true_path = '/bin/true'
elif os.path.exists('/usr/bin/true'):
true_path = '/usr/bin/true'
else:
raise NotImplementedError('Unsupported test platform.')
tests = [(true_path, ''), ('/bin/echo Hello', 'Hello'),
('/bin/echo "Hello"', 'Hello'),
('/bin/echo "Hello World"', 'Hello World'),
('/bin/echo "Hello\nWorld"', 'Hello\nWorld'),
('/bin/echo \'"Hello World"\'', '"Hello World"')]
for cmd, expect in tests:
if check:
output = check_subprocess(cmd)
else:
code, output = run_subprocess(cmd)
self.assertEqual(0, code)
self.assertEqual(expect, output)
def _do_command(self):
"""Wraps base method so we can start/stop redis if needed."""
self._ensure_templates_directory()
redis_name = 'redis-server'
start_redis = run_subprocess('sudo service %s start' % redis_name)[0] != 0
logging.debug('redis-server %s running.',
'IS NOT' if start_redis else 'IS')
try:
if start_redis:
check_subprocess('sudo service %s start' % redis_name)
super(BuildApiDocsCommand, self)._do_command()
finally:
if start_redis:
# pylint: disable=broad-except
try:
check_subprocess('sudo service %s stop' % redis_name)
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping temporary ' + redis_name)
def _do_command(self):
"""Wraps base method so we can start/stop redis if needed."""
self._ensure_templates_directory()
redis_name = 'redis-server'
start_redis = run_subprocess('sudo service %s start' % redis_name)[0] != 0
logging.debug('redis-server %s running.',
'IS NOT' if start_redis else 'IS')
try:
if start_redis:
check_subprocess('sudo service %s start' % redis_name)
super(BuildApiDocsCommand, self)._do_command()
finally:
if start_redis:
# pylint: disable=broad-except
try:
check_subprocess('sudo service %s stop' % redis_name)
except Exception as ex:
maybe_log_exception(
self.name, ex,
'Ignoring exception while stopping temporary ' + redis_name)
def do_run_subprocess_ok(self, check, logfile=None):
if os.path.exists("/bin/true"):
true_path = "/bin/true"
elif os.path.exists("/usr/bin/true"):
true_path = "/usr/bin/true"
else:
raise NotImplementedError("Unsupported test platform.")
tests = [
(true_path, ""),
("/bin/echo Hello", "Hello"),
('/bin/echo "Hello"', "Hello"),
('/bin/echo "Hello World"', "Hello World"),
('/bin/echo "Hello\nWorld"', "Hello\nWorld"),
("/bin/echo '\"Hello World\"'", '"Hello World"'),
]
for cmd, expect in tests:
if logfile:
output = check_subprocesses_to_logfile("Test Logfile", logfile,
[cmd])
elif check:
output = check_subprocess(cmd)
else:
code, output = run_subprocess(cmd)
self.assertEqual(0, code)
if logfile:
self.assertTrue(os.path.exists(logfile))
self.assertIsNone(output)
with io.open(logfile, "r", encoding="utf-8") as stream:
lines = stream.read().split("\n")
self.assertTrue("Spawning" in lines[0])
self.assertTrue("process completed with" in lines[-2])
body = "\n".join(lines[3:-3]).strip()
self.assertEqual(expect, body)
else:
self.assertEqual(expect, output)
def run_git(self, git_dir, command, **kwargs):
"""Wrapper around run_subprocess."""
self.__inject_auth(kwargs)
return run_subprocess(
'git -C "{dir}" {command}'.format(dir=git_dir, command=command), **kwargs
)
def run_git(self, git_dir, command, **kwargs):
"""Wrapper around run_subprocess."""
self.__inject_auth(kwargs)
return run_subprocess(
'git -C "{dir}" {command}'.format(dir=git_dir, command=command),
**kwargs)
