def main():
distribution = sys.argv[1]
branches = local_prep(distribution)
image_filenames = []
for branch_data in branches:
if branch_data.get('debs'):
cache_debs(branch_data['debs'])
for uca in sorted(UCA_POCKETS):
cache_debs(branch_data['debs'], uca)
elif branch_data.get('rpms'):
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
image_filenames.append(fname)
download(url, fname)
# cache get-pip, because upstream network connection fails more
# often than you might imagine.
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
def clone_repo(project):
remote = "%s/%s.git" % (GIT_BASE, project)
# Clear out any existing target directory first, in case of a retry.
try:
shutil.rmtree(os.path.join("/opt/git", project))
except OSError:
pass
# Try to clone the requested git repository.
(status, out) = run_local(["git", "clone", remote, project], status=True, cwd="/opt/git")
# If it claims to have worked, make sure we can list branches.
if status == 0:
(status, moreout) = run_local(["git", "branch", "-a"], status=True, cwd=os.path.join("/opt/git", project))
out = "\n".join((out, moreout))
# If that worked, try resetting to HEAD to make sure it's there.
if status == 0:
(status, moreout) = run_local(
["git", "reset", "--hard", "HEAD"], status=True, cwd=os.path.join("/opt/git", project)
)
out = "\n".join((out, moreout))
# Status of 0 imples all the above worked, 1 means something failed.
return (status, out)
def main():
distribution = sys.argv[1]
branches = local_prep(distribution)
image_filenames = []
for branch_data in branches:
if branch_data.get("debs"):
cache_debs(branch_data["debs"])
for uca in sorted(UCA_POCKETS):
cache_debs(branch_data["debs"], uca)
elif branch_data.get("rpms"):
run_local(["sudo", "yum", "install", "-y", "--downloadonly"] + branch_data["rpms"])
else:
sys.exit("No supported package data found.")
for url in branch_data["images"]:
fname = url.split("/")[-1]
if fname in image_filenames:
continue
image_filenames.append(fname)
download(url, fname)
# cache get-pip, because upstream network connection fails more
# often than you might imagine.
download("https://bootstrap.pypa.io/get-pip.py", "get-pip.py")
def _build_wheelhouse(basedir):
build_wheels_path = os.path.join(basedir, 'tools/build_wheels.sh')
if os.path.isfile(build_wheels_path):
env = os.environ
# Makes output dir for wheelhouse CACHEDIR/wheelhouse
env['WHEELHOUSE'] = os.path.join(CACHEDIR, 'wheelhouse')
run_local(['bash', 'tools/build_wheels.sh'], cwd=DEVSTACK, env=env)
def clone_repo(project):
remote = '%s/%s.git' % (GIT_BASE, project)
# Clear out any existing target directory first, in case of a retry.
try:
shutil.rmtree(os.path.join('/opt/git', project))
except OSError:
pass
# Try to clone the requested git repository.
(status, out) = run_local(['git', 'clone', remote, project],
status=True,
cwd='/opt/git')
# If it claims to have worked, make sure we can list branches.
if status == 0:
(status, moreout) = run_local(['git', 'branch', '-a'],
status=True,
cwd=os.path.join('/opt/git', project))
out = '\n'.join((out, moreout))
# If that worked, try resetting to HEAD to make sure it's there.
if status == 0:
(status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'],
status=True,
cwd=os.path.join('/opt/git', project))
out = '\n'.join((out, moreout))
# Status of 0 imples all the above worked, 1 means something failed.
return (status, out)
def main():
distribution = sys.argv[1]
branches = local_prep(distribution)
image_filenames = []
for branch_data in branches:
if branch_data.get('debs'):
cache_debs(branch_data['debs'])
for uca in sorted(UCA_POCKETS):
cache_debs(branch_data['debs'], uca)
elif branch_data.get('rpms'):
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
image_filenames.append(fname)
download(url, fname)
# cache get-pip, because upstream network connection fails more
# often than you might imagine.
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
def clone_repo(project):
remote = '%s/%s.git' % (GIT_BASE, project)
# Clear out any existing target directory first, in case of a retry.
try:
shutil.rmtree(os.path.join('/opt/git', project))
except OSError:
pass
# Try to clone the requested git repository.
(status, out) = run_local(['git', 'clone', remote, project],
status=True, cwd='/opt/git')
# If it claims to have worked, make sure we can list branches.
if status == 0:
(status, moreout) = run_local(['git', 'branch', '-a'], status=True,
cwd=os.path.join('/opt/git', project))
out = '\n'.join((out, moreout))
# If that worked, try resetting to HEAD to make sure it's there.
if status == 0:
(status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'],
status=True,
cwd=os.path.join('/opt/git', project))
out = '\n'.join((out, moreout))
# Status of 0 imples all the above worked, 1 means something failed.
return (status, out)
def _build_wheelhouse(basedir):
build_wheels_path = os.path.join(basedir, 'tools/build_wheels.sh')
if os.path.isfile(build_wheels_path):
env = os.environ
# Makes output dir for wheelhouse CACHEDIR/wheelhouse
env['WHEELHOUSE'] = os.path.join(CACHEDIR, 'wheelhouse')
run_local(['bash', 'tools/build_wheels.sh'], cwd=DEVSTACK, env=env)
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['git', 'checkout', branch], cwd=DEVSTACK)
run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists('/usr/bin/apt-get'):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists('/usr/bin/rpm'):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = []
for line in open(os.path.join(DEVSTACK, 'stackrc')):
line = line.strip()
if line.startswith('IMAGE_URLS'):
if '#' in line:
line = line[:line.rfind('#')]
if line.endswith(';;'):
line = line[:-2]
line = line.split('=', 1)[1].strip()
if line.startswith('${IMAGE_URLS:-'):
line = line[len('${IMAGE_URLS:-'):]
if line.endswith('}'):
line = line[:-1]
if not line:
continue
if line[0] == line[-1] == '"':
line = line[1:-1]
# Add image to the list to be downloaded, but
# skip downloading giant vmware images
images += [
x.strip() for x in line.split(',')
if not x.strip().endswith('vmdk')
]
branch_data['images'] = images
branches.append(branch_data)
return branches
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['git', 'checkout', branch], cwd=DEVSTACK)
run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists('/usr/bin/apt-get'):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists('/usr/bin/rpm'):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = []
for line in open(os.path.join(DEVSTACK, 'stackrc')):
line = line.strip()
if line.startswith('IMAGE_URLS'):
if '#' in line:
line = line[:line.rfind('#')]
if line.endswith(';;'):
line = line[:-2]
line = line.split('=', 1)[1].strip()
if line.startswith('${IMAGE_URLS:-'):
line = line[len('${IMAGE_URLS:-'):]
if line.endswith('}'):
line = line[:-1]
if not line:
continue
if line[0] == line[-1] == '"':
line = line[1:-1]
# Add image to the list to be downloaded, but
# skip downloading giant vmware images
images += [x.strip() for x in line.split(',')
if not x.strip().endswith('vmdk')]
branch_data['images'] = images
branches.append(branch_data)
return branches
def main():
# TODO(jeblair): use gerrit rest api when available
data = urllib2.urlopen(URL).read()
for line in data.split('\n'):
# We're regex-parsing YAML so that we don't have to depend on the
# YAML module which is not in the stdlib.
m = PROJECT_RE.match(line)
if m:
project = 'https://git.openstack.org/%s' % m.group(1)
print run_local(['git', 'clone', project, m.group(1)],
cwd='/opt/git')
def main():
# TODO(jeblair): use gerrit rest api when available
data = urllib2.urlopen(URL).read()
for line in data.split('\n'):
# We're regex-parsing YAML so that we don't have to depend on the
# YAML module which is not in the stdlib.
m = PROJECT_RE.match(line)
if m:
project = 'https://git.openstack.org/%s' % m.group(1)
print run_local(
['git', 'clone', project, m.group(1)], cwd='/opt/git')
def cache_debs(debs, uca_pocket=None):
"""Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
Archive pocket. If a UCA pocket is specified, it is enabled temporarily
for caching only.
"""
if uca_pocket:
# Note this will install the ubuntu-cloud-keyring package which
# contains the required GPG key.
run_local(["sudo", "add-apt-repository", "-y", "cloud-archive:%s" % uca_pocket])
run_local(["sudo", "apt-get", "update"])
run_local(["sudo", "apt-get", "-y", "-d", "install"] + debs)
if uca_pocket:
run_local(["sudo", "rm", "-f", "/etc/apt/sources.list.d/cloudarchive-%s.list" % uca_pocket])
run_local(["sudo", "apt-get", "update"])
def cache_debs(debs, uca_pocket=None):
"""Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
Archive pocket. If a UCA pocket is specified, it is enabled temporarily
for caching only.
"""
if uca_pocket:
# Note this will install the ubuntu-cloud-keyring package which
# contains the required GPG key.
run_local(['sudo', 'add-apt-repository', '-y',
'cloud-archive:%s' % uca_pocket])
run_local(['sudo', 'apt-get', 'update'])
run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
if uca_pocket:
run_local(['sudo', 'rm', '-f',
'/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
run_local(['sudo', 'apt-get', 'update'])
def git_branches():
branches = []
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
branch = branch.strip()
if not branch.startswith('remotes/origin'):
continue
branches.append(branch)
return branches
def git_branches():
branches = []
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
branch = branch.strip()
if not branch.startswith('remotes/origin'):
continue
branches.append(branch)
return branches
def init_testr():
if not os.path.isdir(os.path.join(TEMPEST_PATH, '.testrepository')):
(status, out) = run_local(['testr', 'init'],
status=True,
cwd=TEMPEST_PATH)
if status != 0:
print("testr init failed with:\n%s' % out")
exit(status)
def git_branches():
branches = []
for branch in run_local(["git", "branch", "-a"], cwd=DEVSTACK).split("\n"):
branch = branch.strip()
if not branch.startswith("remotes/origin"):
continue
branches.append(branch)
return branches
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists('/usr/bin/apt-get'):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'debs')
if not os.path.exists(debdir):
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists('/usr/bin/rpm'):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
branch_data['images'] = images
branches.append(branch_data)
_build_wheelhouse(DEVSTACK)
return branches
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists('/usr/bin/apt-get'):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'debs')
if not os.path.exists(debdir):
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists('/usr/bin/rpm'):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
branch_data['images'] = images
branches.append(branch_data)
_build_wheelhouse(DEVSTACK)
return branches
def _find_images(basedir):
images = []
image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
if os.path.exists(image_tool):
returncode, out = run_local(image_tool, status=True)
if returncode:
print "%s failed" % image_tool
print "Exit: %s, Output: %s" % (returncode, out)
# reset images so we'll fall back
images = []
else:
images = out.split('\n')
return images
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if " -> " in branch:
continue
branch_data = {"name": branch}
print "Branch: ", branch
run_local(["git", "checkout", branch], cwd=DEVSTACK)
run_local(["git", "pull", "--ff-only", "origin"], cwd=DEVSTACK)
if os.path.exists("/usr/bin/apt-get"):
debs = []
debdir = os.path.join(DEVSTACK, "files", "apts")
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment="#")
branch_data["debs"] = debs
if os.path.exists("/usr/bin/rpm"):
rpms = []
rpmdir = os.path.join(DEVSTACK, "files", "rpms")
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment="#")
branch_data["rpms"] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
branch_data["images"] = images
branches.append(branch_data)
return branches
def main():
distribution = sys.argv[1]
if (os.path.exists('/etc/redhat-release') and
open('/etc/redhat-release').read().startswith("CentOS release 6")):
# --downloadonly is provided by the yum-plugin-downloadonly package
# on CentOS 6.x
centos6 = True
run_local(['sudo', 'yum', 'install', '-y', 'yum-plugin-downloadonly'])
else:
centos6 = False
branches = local_prep(distribution)
image_filenames = []
for branch_data in branches:
if branch_data.get('debs'):
cache_debs(branch_data['debs'])
for uca in sorted(UCA_POCKETS):
cache_debs(branch_data['debs'], uca)
elif branch_data.get('rpms'):
if centos6:
# some packages may depend on python-setuptools, which is not
# installed and cannot be reinstalled on CentOS 6.x once yum
# has erased them, so use --skip-broken to avoid aborting; also
# on this platform --downloadonly causes yum to return nonzero
# even when it succeeds, so ignore its exit code
run_local([
'sudo', 'yum', 'install', '-y', '--downloadonly',
'--skip-broken'
] + branch_data['rpms'])
else:
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
image_filenames.append(fname)
download(url, fname)
# cache get-pip, because upstream network connection fails more
# often than you might imagine.
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
def main():
distribution = sys.argv[1]
if (os.path.exists('/etc/redhat-release') and
open('/etc/redhat-release').read().startswith("CentOS release 6")):
# --downloadonly is provided by the yum-plugin-downloadonly package
# on CentOS 6.x
centos6 = True
run_local(['sudo', 'yum', 'install', '-y', 'yum-plugin-downloadonly'])
else:
centos6 = False
branches = local_prep(distribution)
image_filenames = []
for branch_data in branches:
if branch_data.get('debs'):
cache_debs(branch_data['debs'])
for uca in sorted(UCA_POCKETS):
cache_debs(branch_data['debs'], uca)
elif branch_data.get('rpms'):
if centos6:
# some packages may depend on python-setuptools, which is not
# installed and cannot be reinstalled on CentOS 6.x once yum
# has erased them, so use --skip-broken to avoid aborting; also
# on this platform --downloadonly causes yum to return nonzero
# even when it succeeds, so ignore its exit code
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly',
'--skip-broken'] + branch_data['rpms'])
else:
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
image_filenames.append(fname)
download(url, fname)
# cache get-pip, because upstream network connection fails more
# often than you might imagine.
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
def main():
distribution = sys.argv[1]
branches = local_prep(distribution)
image_filenames = {}
for branch_data in branches:
if branch_data.get('debs'):
run_local(['sudo', 'apt-get', '-y', '-d', 'install'] +
branch_data['debs'])
elif branch_data.get('rpms'):
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
run_local(['wget', '-nv', '-c', url,
'-O', os.path.join(CACHEDIR, fname)])
def main():
distribution = sys.argv[1]
branches = local_prep(distribution)
image_filenames = {}
for branch_data in branches:
if branch_data.get('debs'):
run_local(['sudo', 'apt-get', '-y', '-d', 'install'] +
branch_data['debs'])
elif branch_data.get('rpms'):
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
run_local(['wget', '-nv', '-c', url,
'-O', os.path.join(CACHEDIR, fname)])
def download(url, fname):
run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
def populate_testrepository(path):
run_local(['testr', 'load', path], cwd=TEMPEST_PATH)
def download(url, fname):
run_local(["wget", "-nv", "-c", url, "-O", os.path.join(CACHEDIR, fname)])