def handleRecipe(self,
xmlrecipe,
user,
guest=False,
ignore_missing_tasks=False):
if not guest:
recipe = MachineRecipe(ttasks=0)
for xmlguest in xmlrecipe.iter_guests():
guestrecipe = self.handleRecipe(
xmlguest,
user,
guest=True,
ignore_missing_tasks=ignore_missing_tasks)
recipe.guests.append(guestrecipe)
else:
recipe = GuestRecipe(ttasks=0)
recipe.guestname = xmlrecipe.guestname
recipe.guestargs = xmlrecipe.guestargs
recipe.host_requires = xmlrecipe.hostRequires()
recipe.distro_requires = xmlrecipe.distroRequires()
recipe.partitions = xmlrecipe.partitions()
try:
recipe.distro_tree = DistroTree.by_filter(
"%s" % recipe.distro_requires)[0]
except IndexError:
raise BX(
_('No distro tree matches Recipe: %s') %
recipe.distro_requires)
try:
# try evaluating the host_requires, to make sure it's valid
systems = XmlHost.from_string(recipe.host_requires).apply_filter(
System.query)
except StandardError, e:
raise BX(_('Error in hostRequires: %s' % e))
def check_filter(self, filterxml, present=[], absent=[]):
session.flush()
query = XmlHost.from_string(filterxml).apply_filter(System.query)
if present:
self.assertItemsEqual(present,
query.filter(System.id.in_([s.id for s in present])).all())
if absent:
self.assertItemsEqual([],
query.filter(System.id.in_([s.id for s in absent])).all())
def check_filter(self, filterxml, present=[], absent=[]):
session.flush()
query = XmlHost.from_string(filterxml).apply_filter(System.query)
if present:
self.assertItemsEqual(present,
query.filter(System.id.in_([s.id for s in present])).all())
if absent:
self.assertItemsEqual([],
query.filter(System.id.in_([s.id for s in absent])).all())
def provision_virt_recipe(recipe_id):
log.debug('Attempting to provision dynamic virt guest for recipe %s', recipe_id)
session.begin()
try:
recipe = Recipe.by_id(recipe_id)
manager = dynamic_virt.VirtManager(recipe.recipeset.job.owner)
available_flavors = manager.available_flavors()
# We want them in order of smallest to largest, so that we can pick the
# smallest flavor that satisfies the recipe's requirements. Sorting by RAM
# is a decent approximation.
possible_flavors = XmlHost.from_string(recipe.host_requires)\
.filter_openstack_flavors(available_flavors, manager.lab_controller)
if not possible_flavors:
log.debug('No OpenStack flavors matched recipe %s, marking precluded',
recipe.id)
recipe.virt_status = RecipeVirtStatus.precluded
return
possible_flavors = sorted(possible_flavors, key=lambda flavor: flavor.ram)
flavor = possible_flavors[0]
vm_name = '%srecipe-%s' % (
ConfigItem.by_name(u'guest_name_prefix').current_value(u'beaker-'),
recipe.id)
# FIXME can we control use of virtio?
#virtio_possible = True
#if self.recipe.distro_tree.distro.osversion.osmajor.osmajor == "RedHatEnterpriseLinux3":
# virtio_possible = False
vm = manager.create_vm(vm_name, flavor)
vm.instance_created = datetime.utcnow()
try:
recipe.createRepo()
recipe.systems = []
recipe.watchdog = Watchdog()
recipe.resource = vm
recipe.recipeset.lab_controller = manager.lab_controller
recipe.virt_status = RecipeVirtStatus.succeeded
recipe.schedule()
log.info("recipe ID %s moved from Queued to Scheduled by provision_virt_recipe" % recipe.id)
recipe.waiting()
recipe.provision()
log.info("recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe" % recipe.id)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
try:
manager.destroy_vm(vm)
except Exception:
log.exception('Failed to clean up vm %s '
'during provision_virt_recipe, leaked!', vm.instance_id)
# suppress this exception so the original one is not masked
raise exc_type, exc_value, exc_tb
session.commit()
except Exception, e:
log.exception('Error in provision_virt_recipe(%s)', recipe_id)
session.rollback()
# As an added precaution, let's try and avoid this recipe in future
with session.begin():
recipe = Recipe.by_id(recipe_id)
recipe.virt_status = RecipeVirtStatus.failed
def provision_virt_recipe(recipe_id):
log.debug('Attempting to provision dynamic virt guest for recipe %s', recipe_id)
session.begin()
try:
recipe = Recipe.by_id(recipe_id)
manager = dynamic_virt.VirtManager(recipe.recipeset.job.owner)
available_flavors = manager.available_flavors()
# We want them in order of smallest to largest, so that we can pick the
# smallest flavor that satisfies the recipe's requirements. Sorting by RAM
# is a decent approximation.
possible_flavors = XmlHost.from_string(recipe.host_requires)\
.filter_openstack_flavors(available_flavors, manager.lab_controller)
if not possible_flavors:
log.debug('No OpenStack flavors matched recipe %s, marking precluded',
recipe.id)
recipe.virt_status = RecipeVirtStatus.precluded
return
possible_flavors = sorted(possible_flavors, key=lambda flavor: flavor.ram)
flavor = possible_flavors[0]
vm_name = '%srecipe-%s' % (
ConfigItem.by_name(u'guest_name_prefix').current_value(u'beaker-'),
recipe.id)
# FIXME can we control use of virtio?
#virtio_possible = True
#if self.recipe.distro_tree.distro.osversion.osmajor.osmajor == "RedHatEnterpriseLinux3":
# virtio_possible = False
vm = manager.create_vm(vm_name, flavor)
vm.instance_created = datetime.utcnow()
try:
recipe.createRepo()
recipe.clear_candidate_systems()
recipe.watchdog = Watchdog()
recipe.resource = vm
recipe.recipeset.lab_controller = manager.lab_controller
recipe.virt_status = RecipeVirtStatus.succeeded
recipe.schedule()
log.info("recipe ID %s moved from Queued to Scheduled by provision_virt_recipe" % recipe.id)
recipe.waiting()
recipe.provision()
log.info("recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe" % recipe.id)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
try:
manager.destroy_vm(vm)
except Exception:
log.exception('Failed to clean up vm %s '
'during provision_virt_recipe, leaked!', vm.instance_id)
# suppress this exception so the original one is not masked
raise exc_type, exc_value, exc_tb
session.commit()
except Exception, e:
log.exception('Error in provision_virt_recipe(%s)', recipe_id)
session.rollback()
# As an added precaution, let's try and avoid this recipe in future
with session.begin():
recipe = Recipe.by_id(recipe_id)
recipe.virt_status = RecipeVirtStatus.failed
def test_multiple_nonexistent_keys(self):
filter = """
<hostRequires>
<and>
<key_value key="NOTEXIST1" op="=" value="asdf"/>
<key_value key="NOTEXIST2" op="=" value="asdf"/>
</and>
</hostRequires>
"""
query = XmlHost.from_string(filter).apply_filter(System.query)
query.all() # don't care about the results, just that it doesn't break
def test_multiple_nonexistent_keys(self):
filter = """
<hostRequires>
<and>
<key_value key="NOTEXIST1" op="=" value="asdf"/>
<key_value key="NOTEXIST2" op="=" value="asdf"/>
</and>
</hostRequires>
"""
query = XmlHost.from_string(filter).apply_filter(System.query)
query.all() # don't care about the results, just that it doesn't break
def test_arch_inside_or(self):
# The bug was that <arch/> inside <or/> with other conditions would
# produce a cartesian product where the join condition was on one side
# of the OR, leading to a row explosion (similar to the case above).
lc = data_setup.create_labcontroller(fqdn=u'bz1183239.lab')
data_setup.create_system(lab_controller=lc, arch=[u'i386', u'x86_64'])
session.flush()
filter = """
<hostRequires>
<labcontroller value="bz1183239.lab" />
<or>
<system><arch value="i386" /></system>
<hostname op="!=" value="blerch" />
</or>
</hostRequires>
"""
query = XmlHost.from_string(filter).apply_filter(System.query)
self.assertEquals(len(query.all()), 1)
# with the bug this count comes out as 2 instead of 1,
# which doesn't sound so bad...
# but when it's 30575826 instead of 4131, that's bad
self.assertEquals(query.count(), 1)
def test_keyvalue_does_not_cause_duplicate_rows(self):
system = data_setup.create_system()
disk_key = Key.by_name(u'DISK')
system.key_values_int.extend([
Key_Value_Int(disk_key, 30718),
Key_Value_Int(disk_key, 140011),
Key_Value_Int(disk_key, 1048570)])
session.flush()
filter = """
<hostRequires>
<and>
<system><name op="=" value="%s" /></system>
<key_value key="DISK" op=">" value="9000" />
</and>
</hostRequires>
""" % system.fqdn
query = XmlHost.from_string(filter).apply_filter(System.query)
self.assertEquals(len(query.all()), 1)
# with the bug this count comes out as 3 instead of 1,
# which doesn't sound so bad...
# but when it's 926127 instead of 278, that's bad
self.assertEquals(query.count(), 1)
def test_keyvalue_does_not_cause_duplicate_rows(self):
system = data_setup.create_system()
disk_key = Key.by_name(u'DISK')
system.key_values_int.extend([
Key_Value_Int(disk_key, 30718),
Key_Value_Int(disk_key, 140011),
Key_Value_Int(disk_key, 1048570)])
session.flush()
filter = """
<hostRequires>
<and>
<system><name op="=" value="%s" /></system>
<key_value key="DISK" op=">" value="9000" />
</and>
</hostRequires>
""" % system.fqdn
query = XmlHost.from_string(filter).apply_filter(System.query)
self.assertEquals(len(query.all()), 1)
# with the bug this count comes out as 3 instead of 1,
# which doesn't sound so bad...
# but when it's 926127 instead of 278, that's bad
self.assertEquals(query.count(), 1)
def handleRecipe(self, xmlrecipe, user, guest=False, ignore_missing_tasks=False):
if not guest:
recipe = MachineRecipe(ttasks=0)
for xmlguest in xmlrecipe.iter_guests():
guestrecipe = self.handleRecipe(xmlguest, user, guest=True,
ignore_missing_tasks=ignore_missing_tasks)
recipe.guests.append(guestrecipe)
else:
recipe = GuestRecipe(ttasks=0)
recipe.guestname = xmlrecipe.guestname
recipe.guestargs = xmlrecipe.guestargs
recipe.host_requires = xmlrecipe.hostRequires()
recipe.distro_requires = xmlrecipe.distroRequires()
recipe.partitions = xmlrecipe.partitions()
try:
recipe.distro_tree = DistroTree.by_filter("%s" %
recipe.distro_requires)[0]
except IndexError:
raise BX(_('No distro tree matches Recipe: %s') % recipe.distro_requires)
try:
# try evaluating the host_requires, to make sure it's valid
systems = XmlHost.from_string(recipe.host_requires).apply_filter(System.query)
except StandardError, e:
raise BX(_('Error in hostRequires: %s' % e))
def check_filter(self, filterxml, lab_controller=None, present=[], absent=[]):
if lab_controller is None:
lab_controller = self.lc
matched_flavors = XmlHost.from_string(filterxml)\
.filter_openstack_flavors(present + absent, lab_controller)
self.assertItemsEqual(present, matched_flavors)
def check_filter(self, filterxml, virtualisable_expected):
result = XmlHost.from_string(filterxml).virtualisable()
self.assertEquals(result, virtualisable_expected)
def provision_virt_recipe(recipe_id):
log.debug('Attempting to provision dynamic virt guest for recipe %s',
recipe_id)
session.begin()
try:
recipe = Recipe.by_id(recipe_id)
job_owner = recipe.recipeset.job.owner
manager = dynamic_virt.VirtManager(job_owner)
available_flavors = manager.available_flavors()
# We want them in order of smallest to largest, so that we can pick the
# smallest flavor that satisfies the recipe's requirements. Sorting by RAM
# is a decent approximation.
possible_flavors = XmlHost.from_string(recipe.host_requires)\
.filter_openstack_flavors(available_flavors, manager.lab_controller)
if not possible_flavors:
log.info(
'No OpenStack flavors matched recipe %s, marking precluded',
recipe.id)
recipe.virt_status = RecipeVirtStatus.precluded
return
# cheapest flavor has the smallest disk and ram
# id guarantees consistency of our results
flavor = min(possible_flavors,
key=lambda flavor: (flavor.ram, flavor.disk, flavor.id))
vm_name = '%srecipe-%s' % (ConfigItem.by_name(
u'guest_name_prefix').current_value(u'beaker-'), recipe.id)
log.debug('Creating VM named %s as flavor %s', vm_name, flavor)
vm = manager.create_vm(vm_name, flavor)
vm.instance_created = datetime.utcnow()
try:
recipe.createRepo()
recipe.clear_candidate_systems()
recipe.watchdog = Watchdog()
recipe.resource = vm
recipe.recipeset.lab_controller = manager.lab_controller
recipe.virt_status = RecipeVirtStatus.succeeded
recipe.schedule()
log.info(
"recipe ID %s moved from Queued to Scheduled by provision_virt_recipe",
recipe.id)
recipe.waiting()
recipe.provision()
log.info(
"recipe ID %s moved from Scheduled to Waiting by provision_virt_recipe",
recipe.id)
except:
exc_type, exc_value, exc_tb = sys.exc_info()
try:
manager.destroy_vm(vm)
except Exception:
log.exception(
'Failed to clean up VM %s during provision_virt_recipe, leaked!',
vm.instance_id)
# suppress this exception so the original one is not masked
raise exc_type, exc_value, exc_tb
session.commit()
except Exception as e:
log.exception('Error in provision_virt_recipe(%s)', recipe_id)
session.rollback()
# As an added precaution, let's try and avoid this recipe in future
with session.begin():
recipe = Recipe.by_id(recipe_id)
recipe.virt_status = RecipeVirtStatus.failed
finally:
session.close()
def check_filter(self, filterxml, lab_controller=None, present=[], absent=[]):
if lab_controller is None:
lab_controller = self.lc
matched_flavors = XmlHost.from_string(filterxml)\
.filter_openstack_flavors(present + absent, lab_controller)
self.assertItemsEqual(present, matched_flavors)
def check_filter(self, filterxml, virtualisable_expected):
result = XmlHost.from_string(filterxml).virtualisable()
self.assertEquals(result, virtualisable_expected)