def add_artifacts(workspace_dir, globs, dry_run=False, verbose=False): """Find files beneath the workspace_dir and move them to the artifacts. The list of globs can match the full file name, part of a name, or a sub directory: eg: buildvars.json, *.deb, tmp/*.deb. """ workspace_dir = os.path.realpath(workspace_dir) artifacts_dir = os.path.join(workspace_dir, 'artifacts') for root, dirs, files in os.walk(workspace_dir): # create a pseudo-relative path to make glob matches easy. relative = os.path.relpath(root, workspace_dir) if relative == '.': relative = '' if 'artifacts' in dirs: dirs.remove('artifacts') for file_name in files: file_path = os.path.join(root, file_name) file_relative_path = os.path.join(relative, file_name) for glob in globs: if fnmatch.fnmatch(file_relative_path, glob): if verbose: print_now("Adding artifact %s" % file_relative_path) if not dry_run: shutil.move(file_path, artifacts_dir) break
def add_artifacts(workspace_dir, globs, dry_run=False, verbose=False): """Find files beneath the workspace_dir and move them to the artifacts. The list of globs can match the full file name, part of a name, or a sub directory: eg: buildvars.json, *.deb, tmp/*.deb. """ workspace_dir = os.path.realpath(workspace_dir) artifacts_dir = os.path.join(workspace_dir, 'artifacts') for root, dirs, files in os.walk(workspace_dir): # create a pseudo-relative path to make glob matches easy. relative = os.path.relpath(root, workspace_dir) if relative == '.': relative = '' if 'artifacts' in dirs: dirs.remove('artifacts') for file_name in files: file_path = os.path.join(root, file_name) file_relative_path = os.path.join(relative, file_name) for glob in globs: if fnmatch.fnmatch(file_relative_path, glob): if verbose: print_now("Adding artifact %s" % file_relative_path) if not dry_run: shutil.move(file_path, artifacts_dir) break
def list_artifacts(credentials, job_name, build, glob, verbose=False): build_data = get_build_data(JENKINS_URL, credentials, job_name, build) artifacts = find_artifacts(build_data, glob) for artifact in artifacts: if verbose: print_now(artifact.location) else: print_now(artifact.file_name)
def list_artifacts(credentials, job_name, build, glob, verbose=False): build_data = get_build_data(JENKINS_URL, credentials, job_name, build) artifacts = find_artifacts(build_data, glob) for artifact in artifacts: if verbose: print_now(artifact.location) else: print_now(artifact.file_name)
def retain_config(runtime_config, log_directory): if not runtime_config: return False try: shutil.copy(runtime_config, log_directory) return True except IOError: print_now("Failed to copy file. Source: %s Destination: %s" % (runtime_config, log_directory)) return False
def dump_juju_timings(client, log_directory): try: report_path = os.path.join(log_directory, 'juju_command_times.yaml') with open(report_path, 'w') as timing_file: yaml.safe_dump( client.get_juju_timings(), timing_file) timing_file.write('\n') except Exception as e: print_now("Failed to save timings") print_now(str(e))
def main(argv=None): """Run go test against the content of a tarfile.""" returncode = 0 args = parse_args(argv) tarfile_path = args.tarfile tarfile_name = os.path.basename(tarfile_path) version = tarfile_name.split('_')[-1].replace('.tar.gz', '') try: if args.verbose: print_now('Testing juju %s from %s' % (version, tarfile_name)) with temp_dir() as workspace: gopath = os.path.join(workspace, 'gogo') untar_gopath(tarfile_path, gopath, delete=args.remove_tarfile, verbose=args.verbose) returncode = go_test_package(args.package, args.go, gopath, verbose=args.verbose) except Exception as e: print_now(str(e)) print_now(traceback.print_exc()) return 3 return returncode
def main(argv): """Manage list and get files from Juju CI builds.""" try: args, credentials = parse_args(argv) except CredentialsMissing as e: print_now(e) sys.exit(2) try: if args.command == 'list': list_artifacts(credentials, args.job, args.build, args.glob, verbose=args.verbose) elif args.command == 'setup-workspace': setup_workspace(args.path, dry_run=args.dry_run, verbose=args.verbose) except Exception as e: print_now(e) if args.verbose: traceback.print_tb(sys.exc_info()[2]) return 2 if args.verbose: print_now("Done.") return 0
def get_artifacts(credentials, job_name, build, glob, path, archive=False, dry_run=False, verbose=False): full_path = os.path.expanduser(path) if archive: if verbose: print_now('Cleaning %s' % full_path) if not os.path.isdir(full_path): raise ValueError('%s does not exist' % full_path) shutil.rmtree(full_path) os.makedirs(full_path) build_data = get_build_data(JENKINS_URL, credentials, job_name, build) artifacts = find_artifacts(build_data, glob) for artifact in artifacts: local_path = os.path.abspath( os.path.join(full_path, artifact.file_name)) if verbose: print_now('Retrieving %s => %s' % (artifact.location, local_path)) else: print_now(artifact.file_name) if not dry_run: retrieve_artifact(credentials, artifact.location, local_path) return artifacts
def untar_gopath(tarfile_path, gopath, delete=False, verbose=False): """Untar the tarfile to the gopath.""" with temp_dir() as tmp_dir: with tarfile.open(name=tarfile_path, mode='r:gz') as tar: tar.extractall(path=tmp_dir) if verbose: print_now('Extracted the Juju source.') dir_name = os.path.basename(tarfile_path).replace('.tar.gz', '') dir_path = os.path.join(tmp_dir, dir_name) shutil.move(dir_path, gopath) if verbose: print_now('Moved %s to %s' % (dir_name, gopath)) if delete: os.unlink(tarfile_path) if verbose: print_now('Deleted %s' % tarfile_path)
def wait_for_state_server_to_shutdown(host, client, instance_id, timeout=60): print_now("Waiting for port to close on %s" % host) wait_for_port(host, 17070, closed=True, timeout=timeout) print_now("Closed.") try: provider_type = client.env.provider except NoProvider: provider_type = None if provider_type == 'openstack': for ignored in until_timeout(300): if not has_nova_instance(client.env, instance_id): print_now('{} was removed from nova list'.format(instance_id)) break else: raise Exception( '{} was not deleted:'.format(instance_id))
def setup_workspace(workspace_dir, dry_run=False, verbose=False): """Clean the workspace directory and create an artifacts sub directory.""" for root, dirs, files in os.walk(workspace_dir): for name in files: print_now('Removing %s' % name) if not dry_run: os.remove(os.path.join(root, name)) for name in dirs: print_now('Removing %s' % name) if not dry_run: shutil.rmtree(os.path.join(root, name)) artifacts_path = os.path.join(workspace_dir, 'artifacts') print_now('Creating artifacts dir.') if not dry_run: os.mkdir(artifacts_path) # "touch empty" to convince jenkins there is an archive. empty_path = os.path.join(artifacts_path, 'empty') if not dry_run: with open(empty_path, 'w'): pass
def setup_workspace(workspace_dir, dry_run=False, verbose=False): """Clean the workspace directory and create an artifacts sub directory.""" for root, dirs, files in os.walk(workspace_dir): for name in files: print_now('Removing %s' % name) if not dry_run: os.remove(os.path.join(root, name)) for name in dirs: print_now('Removing %s' % name) if not dry_run: shutil.rmtree(os.path.join(root, name)) artifacts_path = os.path.join(workspace_dir, 'artifacts') print_now('Creating artifacts dir.') if not dry_run: os.mkdir(artifacts_path) # "touch empty" to convince jenkins there is an archive. empty_path = os.path.join(artifacts_path, 'empty') if not dry_run: with open(empty_path, 'w'): pass
def get_artifacts(credentials, job_name, build, glob, path, archive=False, dry_run=False, verbose=False): full_path = os.path.expanduser(path) if archive: if verbose: print_now('Cleaning %s' % full_path) if not os.path.isdir(full_path): raise ValueError('%s does not exist' % full_path) shutil.rmtree(full_path) os.makedirs(full_path) build_data = get_build_data(JENKINS_URL, credentials, job_name, build) artifacts = find_artifacts(build_data, glob) for artifact in artifacts: local_path = os.path.abspath( os.path.join(full_path, artifact.file_name)) if verbose: print_now('Retrieving %s => %s' % (artifact.location, local_path)) else: print_now(artifact.file_name) if not dry_run: retrieve_artifact(credentials, artifact.location, local_path) return artifacts
def main(argv): """Manage list and get files from Juju CI builds.""" try: args, credentials = parse_args(argv) except CredentialsMissing as e: print_now(e) sys.exit(2) try: if args.command == 'list': list_artifacts( credentials, args.job, args.build, args.glob, verbose=args.verbose) elif args.command == 'setup-workspace': setup_workspace( args.path, dry_run=args.dry_run, verbose=args.verbose) except Exception as e: print_now(e) if args.verbose: traceback.print_tb(sys.exc_info()[2]) return 2 if args.verbose: print_now("Done.") return 0
def go_test(args, gopath): """Download, build and test a go package.""" goenv = dict(os.environ) goenv["GOPATH"] = gopath go = SubcommandRunner("go", goenv) git = SubcommandRunner("git") final_project = args.project if args.feature_branch: final_project = from_feature_dir(args.project) project_ellipsis = final_project + "/..." directory = os.path.join(gopath, "src", final_project) if args.tsv_path: print_now("Getting and installing godeps") go("get", "-v", "-d", "github.com/rogpeppe/godeps/...") go("install", "github.com/rogpeppe/godeps/...") if args.project_url: print_now("Cloning {} from {}".format(final_project, args.project_url)) git("clone", args.project_url, directory) if args.go_get_all and not (args.project_url and args.merge_url): print_now("Getting {} and dependencies using go".format(args.project)) go("get", "-v", "-d", "-t", project_ellipsis) os.chdir(directory) if args.project_ref: print_now("Switching repository to {}".format(args.project_ref)) git("checkout", args.project_ref) if args.merge_url: print_now("Merging {} ref {}".format(args.merge_url, args.merge_ref)) git("fetch", args.merge_url, args.merge_ref) git("merge", "--no-ff", "-m", "Merged " + args.merge_ref, "FETCH_HEAD") if args.go_get_all: print_now("Updating {} dependencies using go".format(args.project)) go("get", "-v", "-d", "-t", project_ellipsis) if args.dependencies: for dep in args.dependencies: print_now("Getting {} and dependencies using go".format(dep)) go("get", "-v", "-d", dep) if args.tsv_path: tsv_path = os.path.join(gopath, "src", final_project, args.tsv_path) print_now("Getting dependencies using godeps from {}".format(tsv_path)) godeps = SubcommandRunner(os.path.join(gopath, "bin", "godeps"), goenv) godeps("-u", tsv_path) go("build", project_ellipsis) go("test", project_ellipsis)
def go_test(args, gopath): """Download, build and test a go package.""" goenv = dict(os.environ) goenv["GOPATH"] = gopath go = SubcommandRunner("go", goenv) git = SubcommandRunner("git") final_project = args.project if args.feature_branch: final_project = from_feature_dir(args.project) project_ellipsis = final_project + "/..." directory = os.path.join(gopath, "src", final_project) if args.tsv_path: print_now("Getting and installing godeps") go("get", "-v", "-d", "github.com/rogpeppe/godeps/...") go("install", "github.com/rogpeppe/godeps/...") if args.project_url: print_now("Cloning {} from {}".format(final_project, args.project_url)) git("clone", args.project_url, directory) if args.go_get_all and not (args.project_url and args.merge_url): print_now("Getting {} and dependencies using go".format(args.project)) go("get", "-v", "-d", "-t", project_ellipsis) os.chdir(directory) if args.project_ref: print_now("Switching repository to {}".format(args.project_ref)) git("checkout", args.project_ref) if args.merge_url: print_now("Merging {} ref {}".format(args.merge_url, args.merge_ref)) git("fetch", args.merge_url, args.merge_ref) git("merge", "--no-ff", "-m", "Merged " + args.merge_ref, "FETCH_HEAD") if args.go_get_all: print_now("Updating {} dependencies using go".format(args.project)) go("get", "-v", "-d", "-t", project_ellipsis) if args.dependencies: for dep in args.dependencies: print_now("Getting {} and dependencies using go".format(dep)) go("get", "-v", "-d", dep) if args.tsv_path: tsv_path = os.path.join(gopath, "src", final_project, args.tsv_path) print_now("Getting dependencies using godeps from {}".format(tsv_path)) godeps = SubcommandRunner(os.path.join(gopath, "bin", "godeps"), goenv) godeps("-u", tsv_path) go("build", project_ellipsis) go("test", project_ellipsis)
def go_test_package(package, go_cmd, gopath, verbose=False): """Run the package unit tests.""" # Set GOPATH and GOARCH to ensure the go command tests extracted # tarfile using the arch the win-agent is compiled with. The # default go env might be 386 used to create a win client. env = dict(os.environ) env['GOPATH'] = gopath env['GOARCH'] = 'amd64' version_cmd = [go_cmd, 'version'] env_cmd = [go_cmd, 'env'] build_cmd = [go_cmd, 'test', '-i', './...'] test_cmd = [go_cmd, 'test', '-timeout=1200s', './...'] if sys.platform == 'win32': # Ensure OpenSSH is never in the path for win tests. sane_path = [p for p in env['PATH'].split(';') if 'OpenSSH' not in p] env['PATH'] = ';'.join(sane_path) if verbose: print_now('Setting environ Path to:') print_now(env['PATH']) # GZ 2015-04-21: Short-term hack to work around case-insensitive issues env['Path'] = env.pop('PATH') tempdir = tempfile.mkdtemp(prefix="tmp-juju-test", dir=gopath) env['TMP'] = env['TEMP'] = tempdir if verbose: print_now('Setting environ TMP and TEMP to:') print_now(env['TEMP']) build_cmd = ['powershell.exe', '-Command'] + build_cmd test_cmd = ['powershell.exe', '-Command'] + test_cmd version_cmd = ['powershell.exe', '-Command'] + version_cmd env_cmd = ['powershell.exe', '-Command'] + env_cmd package_dir = os.path.join(gopath, 'src', package.replace('/', os.sep)) with WorkingDirectory(package_dir): if verbose: print_now('Go environment information') # We don't care about return code for these run(version_cmd, env=env) run(env_cmd, env=env) if verbose: print_now('Building test dependencies') returncode = run(build_cmd, env=env) if returncode != 0: return returncode if verbose: print_now('Running unit tests in %s' % package) returncode = run(test_cmd, env=env) if verbose: if returncode == 0: print_now('SUCCESS') else: print_now('FAIL') print_now("Killing any lingering mongo processes...") murder_mongo() return returncode
def check_services(client): token = env_token(client.env.environment) client.set_config('dummy-source', {'token': token}) print_now("checking services in " + client.env.environment) check_token(client, token)
def check_services(client): token = env_token(client.env.environment) client.set_config('dummy-source', {'token': token}) print_now("checking services in " + client.env.environment) check_token(client, token)