def main():
db = vmdatabase.VMDatabase()
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
for base_image in provider.base_images:
if base_image.min_ready <= 0:
continue
print "Working on base image %s" % base_image.name
flavor = utils.get_flavor(client, base_image.min_ram)
print "Found flavor", flavor
branches = local_prep(base_image.name)
remote_base_image = client.images.find(name=base_image.external_id)
timestamp = int(time.time())
remote_snap_image_name = (
'%sdevstack-%s-%s.template.openstack.org' %
(DEVSTACK_GATE_PREFIX, base_image.name, str(timestamp)))
remote_snap_image = build_image(provider, client, base_image,
remote_base_image, flavor,
remote_snap_image_name, branches,
timestamp)
def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
else:
jenkins = None
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
error = False
for machine in provider.machines:
if machine.state != vmdatabase.READY:
continue
print 'Checking machine', machine.name
try:
check_machine(jenkins, machine)
except:
error = True
traceback.print_exc()
utils.update_stats(provider)
if error:
sys.exit(1)
def main():
db = vmdatabase.VMDatabase()
result = db.getResult(RESULT_ID)
value = RESULTS[RESULT]
# This gets called with an argument of 'timeout' after every run,
# regardless of whether a timeout occured; so in that case, only
# set the result to timeout if there is not already a result.
if not (value == vmdatabase.RESULT_TIMEOUT and result.result):
result.setResult(value)
def main(threshold, stat_file):
db = vmdatabase.VMDatabase()
ready = vmdatabase.READY
ready_nodes = []
for provider in db.getProviders():
for base_image in provider.base_images:
ready_nodes = [x for x in base_image.machines if x.state == ready]
ready_count = len(ready_nodes)
set_vm_state(ready_count, stat_file)
print "Number of slaves available: %s\n" % ready_count
if ready_count < threshold:
sys.exit(1)
def main():
db = vmdatabase.VMDatabase()
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
if not SKIP_DEVSTACK_GATE_JENKINS:
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
else:
jenkins = None
try:
machine = db.getMachineByJenkinsName(NODE_NAME)
except Exception:
utils.log.debug("Unable to find node: %s" % NODE_NAME)
return
utils.log.debug("Used ID: %s old state: %s build:%s" %
(machine.id, machine.state, BUILD_URL))
machine.state = vmdatabase.USED
if jenkins:
if machine.jenkins_name:
if jenkins.node_exists(machine.jenkins_name):
config = jenkins.get_node_config(machine.jenkins_name)
old = None
m = LABEL_RE.search(config)
if m:
old = m.group(1)
config = LABEL_RE.sub('<label>devstack-used</label>', config)
for i in range(3):
try:
jenkins.reconfig_node(machine.jenkins_name, config)
except:
if i == 2:
utils.log.exception("Unable to relabel ID: %s" %
machine.id)
raise
time.sleep(5)
utils.log.debug(
"Relabeled ID: %s old label: %s new label: %s" %
(machine.id, old, 'devstack-used'))
utils.update_stats(machine.base_image.provider)
def main():
db = vmdatabase.VMDatabase()
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
machine = db.getMachineByJenkinsName(NODE_NAME)
machine.state = vmdatabase.USED
if machine.jenkins_name:
if jenkins.node_exists(machine.jenkins_name):
config = jenkins.get_node_config(machine.jenkins_name)
config = LABEL_RE.sub('<label>devstack-used</label>', config)
jenkins.reconfig_node(machine.jenkins_name, config)
def main():
db = vmdatabase.VMDatabase()
try:
machine = db.getMachineByJenkinsName(NODE_NAME)
except Exception:
utils.log.debug("Unable to find node: %s" % NODE_NAME)
return
if machine.state != vmdatabase.HOLD:
utils.log.debug("Set deleted ID: %s old state: %s build: %s" % (
machine.id, machine.state, BUILD_URL))
machine.state = vmdatabase.DELETE
else:
utils.log.debug("Hold ID: %s old state: %s build: %s" % (
machine.id, machine.state, BUILD_URL))
try:
utils.update_stats(machine.base_image.provider)
if UPSTREAM_BUILD_URL:
fd = urllib.urlopen(UPSTREAM_BUILD_URL + 'api/json')
data = json.load(fd)
result = data['result']
if statsd and result == 'SUCCESS':
dt = int(data['duration'])
key = 'devstack.job.%s' % UPSTREAM_JOB_NAME
statsd.timing(key + '.runtime', dt)
statsd.incr(key + '.builds')
key += '.%s' % UPSTREAM_BRANCH
statsd.timing(key + '.runtime', dt)
statsd.incr(key + '.builds')
key += '.%s' % machine.base_image.provider.name
statsd.timing(key + '.runtime', dt)
statsd.incr(key + '.builds')
except:
print "Error getting build information"
traceback.print_exc()
def main():
db = vmdatabase.VMDatabase()
machine = db.getMachine(NODE_ID)
stat, out = commands.getstatusoutput(
"ssh -p 29418 review.openstack.org gerrit" +
"query --format=JSON change:%s" %
os.environ['GERRIT_CHANGE_NUMBER'])
data = json.loads(out.split('\n')[0])
username = data['owner']['username']
f = urllib2.urlopen('https://launchpad.net/~%s/+sshkeys' % username)
keys = f.read()
tmp = tempfile.NamedTemporaryFile(delete=False)
try:
tmp.write("""#!/bin/bash
chmod u+w ~/.ssh/authorized_keys
cat <<EOF >>~/.ssh/authorized_keys
""")
tmp.write(keys)
tmp.write("\nEOF\n")
tmp.close()
stat, out = commands.getstatusoutput("scp %s %s:/var/tmp/keys.sh" %
(tmp.name, machine.ip))
if stat:
print out
raise Exception("Unable to copy keys")
stat, out = commands.getstatusoutput(
"ssh %s /bin/sh /var/tmp/keys.sh" % machine.ip)
if stat:
print out
raise Exception("Unable to add keys")
finally:
os.unlink(tmp.name)
machine.user = username
print "Added %s to authorized_keys on %s" % (username, machine.ip)
def main():
db = vmdatabase.VMDatabase()
node = db.getMachineForUse(IMAGE_NAME)
if not node:
raise Exception("No ready nodes")
job_name = os.environ.get('JOB_NAME', None)
build_number = os.environ.get('BUILD_NUMBER', None)
gerrit_change = os.environ.get('GERRIT_CHANGE_NUMBER', None)
gerrit_patchset = os.environ.get('GERRIT_PATCHSET_NUMBER', None)
if job_name and build_number and gerrit_change and gerrit_patchset:
result = node.newResult(job_name, build_number, gerrit_change,
gerrit_patchset)
else:
result = None
print "NODE_IP_ADDR=%s" % node.ip
print "NODE_PROVIDER=%s" % node.base_image.provider.name
print "NODE_ID=%s" % node.id
if result:
print "RESULT_ID=%s" % result.id
def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
else:
jenkins = None
print 'Known machines (start):'
db.print_state()
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
flavor = utils.get_flavor(client, 1024)
print "Found flavor", flavor
error = False
now = time.time()
for machine in provider.machines:
# Normally, reap machines that have sat in their current state
# for 24 hours, unless that state is READY.
if (REAP_ALL_SERVERS
or (machine.state != vmdatabase.READY
and now - machine.state_time > MACHINE_LIFETIME)
or machine.state == vmdatabase.DELETE):
print 'Deleting machine', machine.name
try:
delete_machine(jenkins, client, machine)
except:
error = True
traceback.print_exc()
provider_min_ready = 0
for base_image in provider.base_images:
provider_min_ready += base_image.min_ready
for snap_image in base_image.snapshot_images:
# Normally, reap images that have sat in their current state
# for 24 hours, unless the image is the current snapshot
if (REAP_ALL_IMAGES
or (snap_image != base_image.current_snapshot
and now - snap_image.state_time > MACHINE_LIFETIME)):
print 'Deleting image', snap_image.name
try:
delete_image(client, snap_image)
except:
error = True
traceback.print_exc()
# Make sure the provider has enough headroom for the min_ready
# of all base images, deleting used serverss if needed.
overcommitment = ((len(provider.machines) - len(provider.ready_machines) +
provider_min_ready) - provider.max_servers)
while overcommitment > 0:
print 'Overcommitted by %s machines' % overcommitment
last_overcommitment = overcommitment
for machine in provider.machines:
if machine.state == vmdatabase.READY:
continue
if machine.state == vmdatabase.BUILDING:
continue
if machine.state == vmdatabase.HOLD:
continue
print 'Deleting machine', machine.name
try:
delete_machine(jenkins, client, machine)
overcommitment -= 1
except:
error = True
traceback.print_exc()
if overcommitment == last_overcommitment:
raise Exception("Unable to reduce overcommitment")
last_overcommitment = overcommitment
print
print 'Known machines (end):'
db.print_state()
if error:
sys.exit(1)
def main():
db = vmdatabase.VMDatabase()
jenkins = None
credentials_id = None
if not SKIP_DEVSTACK_GATE_JENKINS:
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
if config.has_option('jenkins', 'credentials_id'):
credentials_id = config.get('jenkins', 'credentials_id')
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
last_name = ''
error_counts = {}
error = False
for base_image in provider.base_images:
snap_image = base_image.current_snapshot
if not snap_image:
continue
print "Working on image %s" % snap_image.name
flavor = utils.get_flavor(client, base_image.min_ram)
print "Found flavor", flavor
remote_snap_image = client.images.get(snap_image.external_id)
print "Found image", remote_snap_image
num_to_launch = calculate_deficit(provider, base_image)
for i in range(num_to_launch):
try:
server, machine = launch_node(client, snap_image,
remote_snap_image, flavor,
last_name)
last_name = machine.name
except:
traceback.print_exc()
error = True
while True:
utils.update_stats(provider)
building_machines = provider.building_machines
if not building_machines:
print "No more machines are building, finished."
break
print "Waiting on %s machines" % len(building_machines)
for machine in building_machines:
try:
check_machine(jenkins, client, machine, error_counts,
credentials_id)
except:
traceback.print_exc()
print "Abandoning machine %s" % machine.id
utils.log.exception("Abandoning ID: %s" % machine.id)
machine.state = vmdatabase.ERROR
error = True
db.commit()
time.sleep(3)
if error:
sys.exit(1)
def setUp(self):
super(TestVMDatabase, self).setUp()
self.db = vmdatabase.VMDatabase(':memory:')
def main():
db = vmdatabase.VMDatabase()
if not SKIP_DEVSTACK_GATE_JENKINS:
config = ConfigParser.ConfigParser()
config.read(DEVSTACK_GATE_SECURE_CONFIG)
jenkins = myjenkins.Jenkins(config.get('jenkins', 'server'),
config.get('jenkins', 'user'),
config.get('jenkins', 'apikey'))
jenkins.get_info()
else:
jenkins = None
print 'Known machines (start):'
db.print_state()
provider = db.getProvider(PROVIDER_NAME)
print "Working with provider %s" % provider.name
client = utils.get_client(provider)
flavor = utils.get_flavor(client, 1024)
print "Found flavor", flavor
error = False
now = time.time()
for machine in provider.machines:
# Normally, reap machines that have sat in their current state
# for 24 hours, unless that state is READY.
if (REAP_ALL_SERVERS
or (machine.state != vmdatabase.READY
and now - machine.state_time > MACHINE_LIFETIME)
or machine.state == vmdatabase.DELETE
or machine.state == vmdatabase.ERROR):
print 'Deleting machine', machine.name
try:
delete_machine(jenkins, client, machine)
except:
error = True
traceback.print_exc()
provider_min_ready = 0
for base_image in provider.base_images:
provider_min_ready += base_image.min_ready
for snap_image in base_image.snapshot_images:
# Normally, reap images that have sat in their current state
# for 24 hours, unless the image is the current snapshot
if (REAP_ALL_IMAGES
or (snap_image != base_image.current_snapshot
and now - snap_image.state_time > MACHINE_LIFETIME)):
print 'Deleting image', snap_image.name
try:
delete_image(client, snap_image)
except:
error = True
traceback.print_exc()
print
print 'Known machines (end):'
db.print_state()
utils.update_stats(provider)
if error:
sys.exit(1)
def main():
db = vmdatabase.VMDatabase()
machine = db.getMachineByJenkinsName(NODE_NAME)
machine.state = vmdatabase.DELETE
def setUp(self):
self.db = vmdatabase.VMDatabase(':memory:')