def add_artifacts(workspace_dir, globs, dry_run=False, verbose=False):
"""Find files beneath the workspace_dir and move them to the artifacts.
The list of globs can match the full file name, part of a name, or
a sub directory: eg: buildvars.json, *.deb, tmp/*.deb.
"""
workspace_dir = os.path.realpath(workspace_dir)
artifacts_dir = os.path.join(workspace_dir, 'artifacts')
for root, dirs, files in os.walk(workspace_dir):
# create a pseudo-relative path to make glob matches easy.
relative = os.path.relpath(root, workspace_dir)
if relative == '.':
relative = ''
if 'artifacts' in dirs:
dirs.remove('artifacts')
for file_name in files:
file_path = os.path.join(root, file_name)
file_relative_path = os.path.join(relative, file_name)
for glob in globs:
if fnmatch.fnmatch(file_relative_path, glob):
if verbose:
print_now("Adding artifact %s" % file_relative_path)
if not dry_run:
shutil.move(file_path, artifacts_dir)
break
def add_artifacts(workspace_dir, globs, dry_run=False, verbose=False):
"""Find files beneath the workspace_dir and move them to the artifacts.
The list of globs can match the full file name, part of a name, or
a sub directory: eg: buildvars.json, *.deb, tmp/*.deb.
"""
workspace_dir = os.path.realpath(workspace_dir)
artifacts_dir = os.path.join(workspace_dir, 'artifacts')
for root, dirs, files in os.walk(workspace_dir):
# create a pseudo-relative path to make glob matches easy.
relative = os.path.relpath(root, workspace_dir)
if relative == '.':
relative = ''
if 'artifacts' in dirs:
dirs.remove('artifacts')
for file_name in files:
file_path = os.path.join(root, file_name)
file_relative_path = os.path.join(relative, file_name)
for glob in globs:
if fnmatch.fnmatch(file_relative_path, glob):
if verbose:
print_now("Adding artifact %s" % file_relative_path)
if not dry_run:
shutil.move(file_path, artifacts_dir)
break
def list_artifacts(credentials, job_name, build, glob, verbose=False):
build_data = get_build_data(JENKINS_URL, credentials, job_name, build)
artifacts = find_artifacts(build_data, glob)
for artifact in artifacts:
if verbose:
print_now(artifact.location)
else:
print_now(artifact.file_name)
def list_artifacts(credentials, job_name, build, glob, verbose=False):
build_data = get_build_data(JENKINS_URL, credentials, job_name, build)
artifacts = find_artifacts(build_data, glob)
for artifact in artifacts:
if verbose:
print_now(artifact.location)
else:
print_now(artifact.file_name)
def retain_config(runtime_config, log_directory):
if not runtime_config:
return False
try:
shutil.copy(runtime_config, log_directory)
return True
except IOError:
print_now("Failed to copy file. Source: %s Destination: %s" %
(runtime_config, log_directory))
return False
def dump_juju_timings(client, log_directory):
try:
report_path = os.path.join(log_directory, 'juju_command_times.yaml')
with open(report_path, 'w') as timing_file:
yaml.safe_dump(
client.get_juju_timings(),
timing_file)
timing_file.write('\n')
except Exception as e:
print_now("Failed to save timings")
print_now(str(e))
def main(argv=None):
"""Run go test against the content of a tarfile."""
returncode = 0
args = parse_args(argv)
tarfile_path = args.tarfile
tarfile_name = os.path.basename(tarfile_path)
version = tarfile_name.split('_')[-1].replace('.tar.gz', '')
try:
if args.verbose:
print_now('Testing juju %s from %s' % (version, tarfile_name))
with temp_dir() as workspace:
gopath = os.path.join(workspace, 'gogo')
untar_gopath(tarfile_path,
gopath,
delete=args.remove_tarfile,
verbose=args.verbose)
returncode = go_test_package(args.package,
args.go,
gopath,
verbose=args.verbose)
except Exception as e:
print_now(str(e))
print_now(traceback.print_exc())
return 3
return returncode
def main(argv):
"""Manage list and get files from Juju CI builds."""
try:
args, credentials = parse_args(argv)
except CredentialsMissing as e:
print_now(e)
sys.exit(2)
try:
if args.command == 'list':
list_artifacts(credentials,
args.job,
args.build,
args.glob,
verbose=args.verbose)
elif args.command == 'setup-workspace':
setup_workspace(args.path,
dry_run=args.dry_run,
verbose=args.verbose)
except Exception as e:
print_now(e)
if args.verbose:
traceback.print_tb(sys.exc_info()[2])
return 2
if args.verbose:
print_now("Done.")
return 0
def get_artifacts(credentials,
job_name,
build,
glob,
path,
archive=False,
dry_run=False,
verbose=False):
full_path = os.path.expanduser(path)
if archive:
if verbose:
print_now('Cleaning %s' % full_path)
if not os.path.isdir(full_path):
raise ValueError('%s does not exist' % full_path)
shutil.rmtree(full_path)
os.makedirs(full_path)
build_data = get_build_data(JENKINS_URL, credentials, job_name, build)
artifacts = find_artifacts(build_data, glob)
for artifact in artifacts:
local_path = os.path.abspath(
os.path.join(full_path, artifact.file_name))
if verbose:
print_now('Retrieving %s => %s' % (artifact.location, local_path))
else:
print_now(artifact.file_name)
if not dry_run:
retrieve_artifact(credentials, artifact.location, local_path)
return artifacts
def untar_gopath(tarfile_path, gopath, delete=False, verbose=False):
"""Untar the tarfile to the gopath."""
with temp_dir() as tmp_dir:
with tarfile.open(name=tarfile_path, mode='r:gz') as tar:
tar.extractall(path=tmp_dir)
if verbose:
print_now('Extracted the Juju source.')
dir_name = os.path.basename(tarfile_path).replace('.tar.gz', '')
dir_path = os.path.join(tmp_dir, dir_name)
shutil.move(dir_path, gopath)
if verbose:
print_now('Moved %s to %s' % (dir_name, gopath))
if delete:
os.unlink(tarfile_path)
if verbose:
print_now('Deleted %s' % tarfile_path)
def wait_for_state_server_to_shutdown(host, client, instance_id, timeout=60):
print_now("Waiting for port to close on %s" % host)
wait_for_port(host, 17070, closed=True, timeout=timeout)
print_now("Closed.")
try:
provider_type = client.env.provider
except NoProvider:
provider_type = None
if provider_type == 'openstack':
for ignored in until_timeout(300):
if not has_nova_instance(client.env, instance_id):
print_now('{} was removed from nova list'.format(instance_id))
break
else:
raise Exception(
'{} was not deleted:'.format(instance_id))
def setup_workspace(workspace_dir, dry_run=False, verbose=False):
"""Clean the workspace directory and create an artifacts sub directory."""
for root, dirs, files in os.walk(workspace_dir):
for name in files:
print_now('Removing %s' % name)
if not dry_run:
os.remove(os.path.join(root, name))
for name in dirs:
print_now('Removing %s' % name)
if not dry_run:
shutil.rmtree(os.path.join(root, name))
artifacts_path = os.path.join(workspace_dir, 'artifacts')
print_now('Creating artifacts dir.')
if not dry_run:
os.mkdir(artifacts_path)
# "touch empty" to convince jenkins there is an archive.
empty_path = os.path.join(artifacts_path, 'empty')
if not dry_run:
with open(empty_path, 'w'):
pass
def setup_workspace(workspace_dir, dry_run=False, verbose=False):
"""Clean the workspace directory and create an artifacts sub directory."""
for root, dirs, files in os.walk(workspace_dir):
for name in files:
print_now('Removing %s' % name)
if not dry_run:
os.remove(os.path.join(root, name))
for name in dirs:
print_now('Removing %s' % name)
if not dry_run:
shutil.rmtree(os.path.join(root, name))
artifacts_path = os.path.join(workspace_dir, 'artifacts')
print_now('Creating artifacts dir.')
if not dry_run:
os.mkdir(artifacts_path)
# "touch empty" to convince jenkins there is an archive.
empty_path = os.path.join(artifacts_path, 'empty')
if not dry_run:
with open(empty_path, 'w'):
pass
def get_artifacts(credentials, job_name, build, glob, path,
archive=False, dry_run=False, verbose=False):
full_path = os.path.expanduser(path)
if archive:
if verbose:
print_now('Cleaning %s' % full_path)
if not os.path.isdir(full_path):
raise ValueError('%s does not exist' % full_path)
shutil.rmtree(full_path)
os.makedirs(full_path)
build_data = get_build_data(JENKINS_URL, credentials, job_name, build)
artifacts = find_artifacts(build_data, glob)
for artifact in artifacts:
local_path = os.path.abspath(
os.path.join(full_path, artifact.file_name))
if verbose:
print_now('Retrieving %s => %s' % (artifact.location, local_path))
else:
print_now(artifact.file_name)
if not dry_run:
retrieve_artifact(credentials, artifact.location, local_path)
return artifacts
def main(argv):
"""Manage list and get files from Juju CI builds."""
try:
args, credentials = parse_args(argv)
except CredentialsMissing as e:
print_now(e)
sys.exit(2)
try:
if args.command == 'list':
list_artifacts(
credentials, args.job, args.build, args.glob,
verbose=args.verbose)
elif args.command == 'setup-workspace':
setup_workspace(
args.path, dry_run=args.dry_run, verbose=args.verbose)
except Exception as e:
print_now(e)
if args.verbose:
traceback.print_tb(sys.exc_info()[2])
return 2
if args.verbose:
print_now("Done.")
return 0
def go_test(args, gopath):
"""Download, build and test a go package."""
goenv = dict(os.environ)
goenv["GOPATH"] = gopath
go = SubcommandRunner("go", goenv)
git = SubcommandRunner("git")
final_project = args.project
if args.feature_branch:
final_project = from_feature_dir(args.project)
project_ellipsis = final_project + "/..."
directory = os.path.join(gopath, "src", final_project)
if args.tsv_path:
print_now("Getting and installing godeps")
go("get", "-v", "-d", "github.com/rogpeppe/godeps/...")
go("install", "github.com/rogpeppe/godeps/...")
if args.project_url:
print_now("Cloning {} from {}".format(final_project, args.project_url))
git("clone", args.project_url, directory)
if args.go_get_all and not (args.project_url and args.merge_url):
print_now("Getting {} and dependencies using go".format(args.project))
go("get", "-v", "-d", "-t", project_ellipsis)
os.chdir(directory)
if args.project_ref:
print_now("Switching repository to {}".format(args.project_ref))
git("checkout", args.project_ref)
if args.merge_url:
print_now("Merging {} ref {}".format(args.merge_url, args.merge_ref))
git("fetch", args.merge_url, args.merge_ref)
git("merge", "--no-ff", "-m", "Merged " + args.merge_ref, "FETCH_HEAD")
if args.go_get_all:
print_now("Updating {} dependencies using go".format(args.project))
go("get", "-v", "-d", "-t", project_ellipsis)
if args.dependencies:
for dep in args.dependencies:
print_now("Getting {} and dependencies using go".format(dep))
go("get", "-v", "-d", dep)
if args.tsv_path:
tsv_path = os.path.join(gopath, "src", final_project, args.tsv_path)
print_now("Getting dependencies using godeps from {}".format(tsv_path))
godeps = SubcommandRunner(os.path.join(gopath, "bin", "godeps"), goenv)
godeps("-u", tsv_path)
go("build", project_ellipsis)
go("test", project_ellipsis)
def go_test(args, gopath):
"""Download, build and test a go package."""
goenv = dict(os.environ)
goenv["GOPATH"] = gopath
go = SubcommandRunner("go", goenv)
git = SubcommandRunner("git")
final_project = args.project
if args.feature_branch:
final_project = from_feature_dir(args.project)
project_ellipsis = final_project + "/..."
directory = os.path.join(gopath, "src", final_project)
if args.tsv_path:
print_now("Getting and installing godeps")
go("get", "-v", "-d", "github.com/rogpeppe/godeps/...")
go("install", "github.com/rogpeppe/godeps/...")
if args.project_url:
print_now("Cloning {} from {}".format(final_project, args.project_url))
git("clone", args.project_url, directory)
if args.go_get_all and not (args.project_url and args.merge_url):
print_now("Getting {} and dependencies using go".format(args.project))
go("get", "-v", "-d", "-t", project_ellipsis)
os.chdir(directory)
if args.project_ref:
print_now("Switching repository to {}".format(args.project_ref))
git("checkout", args.project_ref)
if args.merge_url:
print_now("Merging {} ref {}".format(args.merge_url, args.merge_ref))
git("fetch", args.merge_url, args.merge_ref)
git("merge", "--no-ff", "-m", "Merged " + args.merge_ref, "FETCH_HEAD")
if args.go_get_all:
print_now("Updating {} dependencies using go".format(args.project))
go("get", "-v", "-d", "-t", project_ellipsis)
if args.dependencies:
for dep in args.dependencies:
print_now("Getting {} and dependencies using go".format(dep))
go("get", "-v", "-d", dep)
if args.tsv_path:
tsv_path = os.path.join(gopath, "src", final_project, args.tsv_path)
print_now("Getting dependencies using godeps from {}".format(tsv_path))
godeps = SubcommandRunner(os.path.join(gopath, "bin", "godeps"), goenv)
godeps("-u", tsv_path)
go("build", project_ellipsis)
go("test", project_ellipsis)
def go_test_package(package, go_cmd, gopath, verbose=False):
"""Run the package unit tests."""
# Set GOPATH and GOARCH to ensure the go command tests extracted
# tarfile using the arch the win-agent is compiled with. The
# default go env might be 386 used to create a win client.
env = dict(os.environ)
env['GOPATH'] = gopath
env['GOARCH'] = 'amd64'
version_cmd = [go_cmd, 'version']
env_cmd = [go_cmd, 'env']
build_cmd = [go_cmd, 'test', '-i', './...']
test_cmd = [go_cmd, 'test', '-timeout=1200s', './...']
if sys.platform == 'win32':
# Ensure OpenSSH is never in the path for win tests.
sane_path = [p for p in env['PATH'].split(';') if 'OpenSSH' not in p]
env['PATH'] = ';'.join(sane_path)
if verbose:
print_now('Setting environ Path to:')
print_now(env['PATH'])
# GZ 2015-04-21: Short-term hack to work around case-insensitive issues
env['Path'] = env.pop('PATH')
tempdir = tempfile.mkdtemp(prefix="tmp-juju-test", dir=gopath)
env['TMP'] = env['TEMP'] = tempdir
if verbose:
print_now('Setting environ TMP and TEMP to:')
print_now(env['TEMP'])
build_cmd = ['powershell.exe', '-Command'] + build_cmd
test_cmd = ['powershell.exe', '-Command'] + test_cmd
version_cmd = ['powershell.exe', '-Command'] + version_cmd
env_cmd = ['powershell.exe', '-Command'] + env_cmd
package_dir = os.path.join(gopath, 'src', package.replace('/', os.sep))
with WorkingDirectory(package_dir):
if verbose:
print_now('Go environment information')
# We don't care about return code for these
run(version_cmd, env=env)
run(env_cmd, env=env)
if verbose:
print_now('Building test dependencies')
returncode = run(build_cmd, env=env)
if returncode != 0:
return returncode
if verbose:
print_now('Running unit tests in %s' % package)
returncode = run(test_cmd, env=env)
if verbose:
if returncode == 0:
print_now('SUCCESS')
else:
print_now('FAIL')
print_now("Killing any lingering mongo processes...")
murder_mongo()
return returncode
def check_services(client):
token = env_token(client.env.environment)
client.set_config('dummy-source', {'token': token})
print_now("checking services in " + client.env.environment)
check_token(client, token)
def check_services(client):
token = env_token(client.env.environment)
client.set_config('dummy-source', {'token': token})
print_now("checking services in " + client.env.environment)
check_token(client, token)