def test_run_subprocess_fail(self): if os.path.exists('/bin/false'): false_path = '/bin/false' elif os.path.exists('/usr/bin/false'): false_path = '/usr/bin/false' else: raise NotImplementedError('Unsupported test platform.') got, output = run_subprocess(false_path) self.assertEquals((1, ''), (got, output)) got, output = run_subprocess('/bin/ls /abc/def') self.assertNotEquals(0, got) self.assertTrue(output.find('No such file or directory') >= 0)
def test_run_subprocess_fail(self): if os.path.exists("/bin/false"): false_path = "/bin/false" elif os.path.exists("/usr/bin/false"): false_path = "/usr/bin/false" else: raise NotImplementedError("Unsupported test platform.") got, output = run_subprocess(false_path) self.assertEqual((1, ""), (got, output)) got, output = run_subprocess("/bin/ls /abc/def") self.assertNotEqual(0, got) self.assertTrue(output.find("No such file or directory") >= 0)
def test_run_subprocess_fail(self): if os.path.exists('/bin/false'): false_path = '/bin/false' elif os.path.exists('/usr/bin/false'): false_path = '/usr/bin/false' else: raise NotImplementedError('Unsupported test platform.') got, output = run_subprocess(false_path) self.assertEqual((1, ''), (got, output)) got, output = run_subprocess('/bin/ls /abc/def') self.assertNotEqual(0, got) self.assertTrue(output.find('No such file or directory') >= 0)
def do_run_subprocess_ok(self, check, logfile=None): if os.path.exists('/bin/true'): true_path = '/bin/true' elif os.path.exists('/usr/bin/true'): true_path = '/usr/bin/true' else: raise NotImplementedError('Unsupported test platform.') tests = [(true_path, ''), ('/bin/echo Hello', 'Hello'), ('/bin/echo "Hello"', 'Hello'), ('/bin/echo "Hello World"', 'Hello World'), ('/bin/echo "Hello\nWorld"', 'Hello\nWorld'), ('/bin/echo \'"Hello World"\'', '"Hello World"')] for cmd, expect in tests: if logfile: output = check_subprocesses_to_logfile('Test Logfile', logfile, [cmd]) elif check: output = check_subprocess(cmd) else: code, output = run_subprocess(cmd) self.assertEqual(0, code) if logfile: self.assertTrue(os.path.exists(logfile)) self.assertIsNone(output) with io.open(logfile, 'r', encoding='utf-8') as stream: lines = stream.read().split('\n') self.assertTrue('Spawning' in lines[0]) self.assertTrue('process completed with' in lines[-2]) body = '\n'.join(lines[3:-3]).strip() self.assertEqual(expect, body) else: self.assertEqual(expect, output)
def __tag_images(versions_to_tag, project, account, project_images, bom_contents_by_name): images_to_tag = set([]) for bom_version in versions_to_tag: to_tag = [ i for i in __derive_images_from_bom( bom_version, bom_contents_by_name) if i in project_images ] images_to_tag.update(to_tag) for image in images_to_tag: return_code, stdout = run_subprocess( 'gcloud compute images describe' ' --project={project} --account={account} --format=json {image}'. format(project=project, account=account, image=image), echo=False) # Adding labels is idempotent, adding the same label again doesn't break anything. if not return_code: payload_str = stdout.strip() timestamp = json.loads(payload_str)['creationTimestamp'] timestamp = timestamp[:timestamp.index('T')] check_subprocess( 'gcloud compute images add-labels --project={project} --account={account} --labels={key}={timestamp} {image}' .format(project=project, account=account, key=PUBLISHED_TAG_KEY, timestamp=timestamp, image=image))
def check_gcloud(): retcode, _ = run_subprocess('gcloud --version') if not retcode: return sys.stderr.write('ERROR: This program requires gcloud. To obtain gcloud:\n' ' curl https://sdk.cloud.google.com | bash\n') sys.exit(-1)
def try_until_ready(command): while True: retcode, stdout = run_subprocess(command) if not retcode: break if stdout.find('refused') > 0: print('New instance does not seem ready yet...retry in 5s.') else: print(stdout.strip()) print('Retrying in 5s.') time.sleep(5)
def test_run_subprocess_get_pid(self): # See if we can run a job by looking up our job # This is also testing parsing command lines. code, output = run_subprocess('/bin/ps -x') self.assertEquals(0, code) my_pid = '%d ' % os.getpid() candidates = [line for line in output.split('\n') if line.find(my_pid) >= 0 and line.find('python') > 0] if len(candidates) != 1: logging.error('Unexpected output\n%s', output) self.assertEquals(1, len(candidates)) self.assertTrue(candidates[0].find(' python ') > 0)
def test_run_subprocess_get_pid(self): # See if we can run a job by looking up our job # This is also testing parsing command lines. code, output = run_subprocess('/bin/ps -x') self.assertEqual(0, code) my_pid = '%d ' % os.getpid() candidates = [line for line in output.split('\n') if line.find(my_pid) >= 0 and line.find('python') > 0] if len(candidates) != 1: logging.error('Unexpected output\n%s', output) self.assertEqual(1, len(candidates)) self.assertTrue(candidates[0].find(' python ') > 0)
def __gcb_image_exists(self, image_name, version): """Determine if gcb image already exists.""" options = self.options command = [ 'gcloud', 'beta', '--account', options.gcb_service_account, 'artifacts', 'docker', 'images', 'describe', '%s/%s:%s' % (options.artifact_registry, image_name, version) ] exit_code = run_subprocess(' '.join(command), stderr=subprocess.PIPE)[0] return exit_code == 0
def load_halyard_version_commits(self): logging.debug('Fetching existing halyard build versions') retcode, stdout = run_subprocess('gsutil cat ' + self.__versions_url) if not retcode: contents = stdout + '\n' else: if stdout.find('No URLs matched') < 0: raise_and_log_error( ExecutionError('No URLs matched', program='gsutil'), 'Could not fetch "%s": %s' % (self.__versions_url, stdout)) contents = '' logging.warning('%s did not exist. Creating a new one.', self.__versions_url) return contents
def load_halyard_version_commits(self): logging.debug('Fetching existing halyard build versions') retcode, stdout = run_subprocess('gsutil cat ' + self.__versions_url) if not retcode: contents = stdout + '\n' else: if stdout.find('No URLs matched') < 0: raise_and_log_error( ExecutionError('No URLs matched', program='gsutil'), 'Could not fetch "%s": %s' % (self.__versions_url, stdout)) contents = '' logging.warning( '%s did not exist. Creating a new one.', self.__versions_url) return contents
def load_halyard_version_commits(self): logging.debug("Fetching existing halyard build versions") retcode, stdout = run_subprocess("gsutil cat " + self.__versions_url) if not retcode: contents = stdout + "\n" else: if stdout.find("No URLs matched") < 0: raise_and_log_error( ExecutionError("No URLs matched", program="gsutil"), 'Could not fetch "%s": %s' % (self.__versions_url, stdout), ) contents = "" logging.warning("%s did not exist. Creating a new one.", self.__versions_url) return contents
def __write_image_delete_script( possible_versions_to_delete, days_before, project, account, project_images, bom_contents_by_name, ): images_to_delete = set([]) print("Calculating images for {} versions to delete.".format( len(possible_versions_to_delete))) for bom_version in possible_versions_to_delete: deletable = [ i for i in __derive_images_from_bom( bom_version, bom_contents_by_name) if i in project_images ] images_to_delete.update(deletable) delete_script_lines = [] for image in images_to_delete: return_code, stdout = run_subprocess( "gcloud compute images describe" " --project={project} --account={account} --format=json {image}". format(project=project, account=account, image=image), echo=False, ) json_str = "" if return_code: # Some BOMs may refer to service versions without HA images. print( "Lookup for image {image} in project {project} failed, ignoring" .format(image=image, project=project)) continue else: json_str = stdout.strip() payload = json.loads(json_str) if __image_age_days(payload) > days_before: labels = payload.get("labels", None) if not labels or PUBLISHED_TAG_KEY not in labels: line = "gcloud compute images delete --project={project} --account={account} {image} -q".format( project=project, account=account, image=image) delete_script_lines.append(line) delete_script = "\n".join(delete_script_lines) timestamp = "{:%Y%m%d%H%M%S}".format(datetime.datetime.utcnow()) script_name = "delete-images-{}".format(timestamp) with open(script_name, "w") as script: script.write(delete_script) print("Wrote image janitor script to {}".format(script_name))
def build_report(test_controller): """Report on the test results.""" options = test_controller.options citest_log_dir = os.path.join(options.log_dir, 'citest_logs') if not os.path.exists(citest_log_dir): logging.warning('%s does not exist -- no citest logs.', citest_log_dir) return None retcode, stdout = run_subprocess( 'python -m citest.reporting.generate_html_report --index *.journal', shell=True, cwd=citest_log_dir) if retcode != 0: logging.error('Error building report: %s', stdout) logging.info('Logging information is in %s', options.log_dir) return test_controller.build_summary()
def do_run_subprocess_ok(self, check): if os.path.exists('/bin/true'): true_path = '/bin/true' elif os.path.exists('/usr/bin/true'): true_path = '/usr/bin/true' else: raise NotImplementedError('Unsupported test platform.') tests = [(true_path, ''), ('/bin/echo Hello', 'Hello'), ('/bin/echo "Hello"', 'Hello'), ('/bin/echo "Hello World"', 'Hello World'), ('/bin/echo "Hello\nWorld"', 'Hello\nWorld'), ('/bin/echo \'"Hello World"\'', '"Hello World"')] for cmd, expect in tests: if check: output = check_subprocess(cmd) else: code, output = run_subprocess(cmd) self.assertEqual(0, code) self.assertEqual(expect, output)
def _do_command(self): """Wraps base method so we can start/stop redis if needed.""" self._ensure_templates_directory() redis_name = 'redis-server' start_redis = run_subprocess('sudo service %s start' % redis_name)[0] != 0 logging.debug('redis-server %s running.', 'IS NOT' if start_redis else 'IS') try: if start_redis: check_subprocess('sudo service %s start' % redis_name) super(BuildApiDocsCommand, self)._do_command() finally: if start_redis: # pylint: disable=broad-except try: check_subprocess('sudo service %s stop' % redis_name) except Exception as ex: maybe_log_exception( self.name, ex, 'Ignoring exception while stopping temporary ' + redis_name)
def do_run_subprocess_ok(self, check, logfile=None): if os.path.exists("/bin/true"): true_path = "/bin/true" elif os.path.exists("/usr/bin/true"): true_path = "/usr/bin/true" else: raise NotImplementedError("Unsupported test platform.") tests = [ (true_path, ""), ("/bin/echo Hello", "Hello"), ('/bin/echo "Hello"', "Hello"), ('/bin/echo "Hello World"', "Hello World"), ('/bin/echo "Hello\nWorld"', "Hello\nWorld"), ("/bin/echo '\"Hello World\"'", '"Hello World"'), ] for cmd, expect in tests: if logfile: output = check_subprocesses_to_logfile("Test Logfile", logfile, [cmd]) elif check: output = check_subprocess(cmd) else: code, output = run_subprocess(cmd) self.assertEqual(0, code) if logfile: self.assertTrue(os.path.exists(logfile)) self.assertIsNone(output) with io.open(logfile, "r", encoding="utf-8") as stream: lines = stream.read().split("\n") self.assertTrue("Spawning" in lines[0]) self.assertTrue("process completed with" in lines[-2]) body = "\n".join(lines[3:-3]).strip() self.assertEqual(expect, body) else: self.assertEqual(expect, output)
def run_git(self, git_dir, command, **kwargs): """Wrapper around run_subprocess.""" self.__inject_auth(kwargs) return run_subprocess( 'git -C "{dir}" {command}'.format(dir=git_dir, command=command), **kwargs )
def run_git(self, git_dir, command, **kwargs): """Wrapper around run_subprocess.""" self.__inject_auth(kwargs) return run_subprocess( 'git -C "{dir}" {command}'.format(dir=git_dir, command=command), **kwargs)