def _sanity_check_generate_control(is_server, client_control_file, kernels, upload_kernel_config): """ Sanity check some of the parameters to generate_control(). This exists as its own function so that site_control_file may call it as well from its own generate_control(). :raise ValidationError if any of the parameters do not make sense. """ if is_server and client_control_file: raise model_logic.ValidationError( {'tests': 'You cannot run server tests at the same time ' 'as directly supplying a client-side control file.'}) if kernels: # make sure that kernel is a list of dictionarions with at least # the 'version' key in them kernel_error = model_logic.ValidationError( {'kernel': 'The kernel parameter must be a sequence of ' 'dictionaries containing at least the "version" key ' '(got: %r)' % kernels}) try: iter(kernels) except TypeError: raise kernel_error for kernel_info in kernels: if (not isinstance(kernel_info, dict) or 'version' not in kernel_info): raise kernel_error if upload_kernel_config and not is_server: raise model_logic.ValidationError( {'upload_kernel_config': 'Cannot use upload_kernel_config ' 'with client side tests'})
def create_new_job(owner, options, host_objects, profiles, metahost_objects, atomic_group=None): labels_by_name = dict((label.name, label) for label in models.Label.objects.all()) all_host_objects = host_objects + metahost_objects metahost_counts = _get_metahost_counts(metahost_objects) dependencies = options.get('dependencies', []) synch_count = options.get('synch_count') if atomic_group: check_atomic_group_create_job( synch_count, host_objects, metahost_objects, dependencies, atomic_group, labels_by_name) else: if synch_count is not None and synch_count > len(all_host_objects): raise model_logic.ValidationError( {'hosts': 'only %d hosts provided for job with synch_count = %d' % (len(all_host_objects), synch_count)}) atomic_hosts = models.Host.objects.filter( id__in=[host.id for host in host_objects], labels__atomic_group=True) unusable_host_names = [host.hostname for host in atomic_hosts] if unusable_host_names: raise model_logic.ValidationError( {'hosts': 'Host(s) "%s" are atomic group hosts but no ' 'atomic group was specified for this job.' % (', '.join(unusable_host_names),)}) check_for_duplicate_hosts(host_objects) check_job_dependencies(host_objects, dependencies) options['dependencies'] = [labels_by_name[label_name] for label_name in dependencies] for label in metahost_objects + options['dependencies']: if label.atomic_group and not atomic_group: raise model_logic.ValidationError( {'atomic_group_name': 'Dependency %r requires an atomic group but no ' 'atomic_group_name or meta_host in an atomic group was ' 'specified for this job.' % label.name}) elif (label.atomic_group and label.atomic_group.name != atomic_group.name): raise model_logic.ValidationError( {'atomic_group_name': 'meta_hosts or dependency %r requires atomic group ' '%r instead of the supplied atomic_group_name=%r.' % (label.name, label.atomic_group.name, atomic_group.name)}) job = models.Job.create(owner=owner, options=options, hosts=all_host_objects) job.queue(all_host_objects, profiles=profiles, atomic_group=atomic_group, is_template=options.get('is_template', False)) return job.id
def delete_saved_queries(id_list): user = afe_models.User.current_user().login query = models.SavedQuery.objects.filter(id__in=id_list, owner=user) if query.count() == 0: raise model_logic.ValidationError( 'No such queries found for this user') query.delete()
def prepare_generate_control_file(tests, kernel, label, profilers): test_objects = [models.Test.smart_get(test) for test in tests] profiler_objects = [models.Profiler.smart_get(profiler) for profiler in profilers] # ensure tests are all the same type try: test_type = get_consistent_value(test_objects, 'test_type') except InconsistencyException as exc: test1 = exc.args[0] test2 = exc.args[1] raise model_logic.ValidationError( {'tests': 'You cannot run both server and client-side tests together (tests %s and %s differ)' % (test1.name, test2.name)}) is_server = (test_type == model_attributes.TestTypes.SERVER) if test_objects: synch_count = max(test.sync_count for test in test_objects) else: synch_count = 1 if label: label = models.Label.smart_get(label) dependencies = set(label.name for label in models.Label.objects.filter(test__in=test_objects)) cf_info = dict(is_server=is_server, synch_count=synch_count, dependencies=list(dependencies)) return cf_info, test_objects, profiler_objects, label
def check_modify_host_locking(host, update_data): """ Checks when locking/unlocking has been requested if the host is already locked/unlocked. :param host: models.Host object to be modified :param update_data: A dictionary with the changes to make to the host. """ locked = update_data.get('locked', None) if locked is not None: if locked and host.locked: raise model_logic.ValidationError({ 'locked': 'Host already locked by %s on %s.' % (host.locked_by, host.lock_time)}) if not locked and not host.locked: raise model_logic.ValidationError({ 'locked': 'Host already unlocked.'})
def check_for_duplicate_hosts(host_objects): host_ids = set() duplicate_hostnames = set() for host in host_objects: if host.id in host_ids: duplicate_hostnames.add(host.hostname) host_ids.add(host.id) if duplicate_hostnames: raise model_logic.ValidationError( {'hosts': 'Duplicate hosts: %s' % ', '.join(duplicate_hostnames)})
def host_add_labels(id, labels): labels = models.Label.smart_get_bulk(labels) host = models.Host.smart_get(id) platforms = [label.name for label in labels if label.platform] if len(platforms) > 1: raise model_logic.ValidationError( {'labels': 'Adding more than one platform label: %s' % ', '.join(platforms)}) if len(platforms) == 1: models.Host.check_no_platform([host]) host.labels.add(*labels)
def check_modify_host(update_data): """ Sanity check modify_host* requests. @param update_data: A dictionary with the changes to make to a host or hosts. """ # Only the scheduler (monitor_db) is allowed to modify Host status. # Otherwise race conditions happen as a hosts state is changed out from # beneath tasks being run on a host. if 'status' in update_data: raise model_logic.ValidationError( {'status': 'Host status can not be modified by the frontend.'})
def prepare_generate_control_file(tests, kernel, label, profilers): test_objects = [models.Test.smart_get(test) for test in tests] profiler_objects = [models.Profiler.smart_get(profiler) for profiler in profilers] # ensure tests are all the same type try: test_type = get_consistent_value(test_objects, 'test_type') except InconsistencyException, exc: test1, test2 = exc.args raise model_logic.ValidationError( {'tests': 'You cannot run both server- and client-side ' 'tests together (tests %s and %s differ' % ( test1.name, test2.name)})
def check_no_platform(cls, hosts): Host.objects.populate_relationships(hosts, Label, 'label_list') errors = [] for host in hosts: platforms = [label.name for label in host.label_list if label.platform] if platforms: # do a join, just in case this host has multiple platforms, # we'll be able to see it errors.append('Host %s already has a platform: %s' % ( host.hostname, ', '.join(platforms))) if errors: raise model_logic.ValidationError({'labels': '; '.join(errors)})
def check_abort_synchronous_jobs(host_queue_entries): # ensure user isn't aborting part of a synchronous autoserv execution count_per_execution = {} for queue_entry in host_queue_entries: key = _execution_key_for(queue_entry) count_per_execution.setdefault(key, 0) count_per_execution[key] += 1 for queue_entry in host_queue_entries: if not queue_entry.execution_subdir: continue execution_count = count_per_execution[_execution_key_for(queue_entry)] if execution_count < queue_entry.job.synch_count: raise model_logic.ValidationError( {'': 'You cannot abort part of a synchronous job execution ' '(%d/%s), %d included, %d expected' % (queue_entry.job.id, queue_entry.execution_subdir, execution_count, queue_entry.job.synch_count)})
def create_one_time_host(hostname): query = Host.objects.filter(hostname=hostname) if query.count() == 0: host = Host(hostname=hostname, invalid=True) host.do_validate() else: host = query[0] if not host.invalid: raise model_logic.ValidationError({ 'hostname' : '%s already exists in the autotest DB. ' 'Select it rather than entering it as a one time ' 'host.' % hostname }) host.protection = host_protections.Protection.DO_NOT_REPAIR host.locked = False host.save() host.clean_object() return host
def check_job_dependencies(host_objects, job_dependencies): """ Check that a set of machines satisfies a job's dependencies. host_objects: list of models.Host objects job_dependencies: list of names of labels """ # check that hosts satisfy dependencies host_ids = [host.id for host in host_objects] hosts_in_job = models.Host.objects.filter(id__in=host_ids) ok_hosts = hosts_in_job for index, dependency in enumerate(job_dependencies): ok_hosts = ok_hosts.filter(labels__name=dependency) failing_hosts = (set(host.hostname for host in host_objects) - set(host.hostname for host in ok_hosts)) if failing_hosts: raise model_logic.ValidationError( {'hosts': 'Host(s) failed to meet job dependencies (' + (', '.join(job_dependencies)) + '): ' + (', '.join(failing_hosts))})
def host_add_labels(id, labels): """ Add labels to host. :param id: Host identification. :param labels: Sequence of labels. :return: None. """ labels = models.Label.smart_get_bulk(labels) host = models.Host.smart_get(id) platforms = [label.name for label in labels if label.platform] if len(platforms) > 1: raise model_logic.ValidationError( {'labels': 'Adding more than one platform label: %s' % ', '.join(platforms)}) if len(platforms) == 1: models.Host.check_no_platform([host]) host.labels.add(*labels)
def create_job_common(name, priority, control_type, control_file=None, hosts=(), meta_hosts=(), one_time_hosts=(), atomic_group_name=None, synch_count=None, is_template=False, timeout=None, max_runtime_hrs=None, run_verify=True, email_list='', dependencies=(), reboot_before=None, reboot_after=None, parse_failed_repair=None, hostless=False, keyvals=None, drone_set=None, parameterized_job=None): """ Common code between creating "standard" jobs and creating parameterized jobs """ user = models.User.current_user() owner = user.login # Convert metahost names to lower case, to avoid case sensitivity issues meta_hosts = [meta_host.lower() for meta_host in meta_hosts] # input validation if not (hosts or meta_hosts or one_time_hosts or atomic_group_name or hostless): raise model_logic.ValidationError({ 'arguments': "You must pass at least one of 'hosts', " "'meta_hosts', 'one_time_hosts', " "'atomic_group_name', or 'hostless'" }) if hostless: if hosts or meta_hosts or one_time_hosts or atomic_group_name: raise model_logic.ValidationError( {'hostless': 'Hostless jobs cannot include any hosts!'}) server_type = models.Job.ControlType.get_string( models.Job.ControlType.SERVER) if control_type != server_type: raise model_logic.ValidationError({ 'control_type': 'Hostless jobs cannot use client-side ' 'control files' }) labels_by_name = dict( (label.name.lower(), label) for label in models.Label.objects.all()) atomic_groups_by_name = dict( (ag.name.lower(), ag) for ag in models.AtomicGroup.objects.all()) # Schedule on an atomic group automagically if one of the labels given # is an atomic group label and no explicit atomic_group_name was supplied. if not atomic_group_name: for label_name in meta_hosts or []: label = labels_by_name.get(label_name) if label and label.atomic_group: atomic_group_name = label.atomic_group.name break # convert hostnames & meta hosts to host/label objects host_objects = models.Host.smart_get_bulk(hosts) metahost_objects = [] for label_name in meta_hosts or []: if label_name in labels_by_name: label = labels_by_name[label_name] metahost_objects.append(label) elif label_name in atomic_groups_by_name: # If given a metahost name that isn't a Label, check to # see if the user was specifying an Atomic Group instead. atomic_group = atomic_groups_by_name[label_name] if atomic_group_name and atomic_group_name != atomic_group.name: raise model_logic.ValidationError({ 'meta_hosts': ('Label "%s" not found. If assumed to be an ' 'atomic group it would conflict with the ' 'supplied atomic group "%s".' % (label_name, atomic_group_name)) }) atomic_group_name = atomic_group.name else: raise model_logic.ValidationError( {'meta_hosts': 'Label "%s" not found' % label_name}) # Create and sanity check an AtomicGroup object if requested. if atomic_group_name: if one_time_hosts: raise model_logic.ValidationError({ 'one_time_hosts': 'One time hosts cannot be used with an Atomic Group.' }) atomic_group = models.AtomicGroup.smart_get(atomic_group_name) if synch_count and synch_count > atomic_group.max_number_of_machines: raise model_logic.ValidationError({ 'atomic_group_name': 'You have requested a synch_count (%d) greater than the ' 'maximum machines in the requested Atomic Group (%d).' % (synch_count, atomic_group.max_number_of_machines) }) else: atomic_group = None for host in one_time_hosts or []: this_host = models.Host.create_one_time_host(host) host_objects.append(this_host) options = dict(name=name, priority=priority, control_file=control_file, control_type=control_type, is_template=is_template, timeout=timeout, max_runtime_hrs=max_runtime_hrs, synch_count=synch_count, run_verify=run_verify, email_list=email_list, dependencies=dependencies, reboot_before=reboot_before, reboot_after=reboot_after, parse_failed_repair=parse_failed_repair, keyvals=keyvals, drone_set=drone_set, parameterized_job=parameterized_job) return create_new_job(owner=owner, options=options, host_objects=host_objects, metahost_objects=metahost_objects, atomic_group=atomic_group)
def check_atomic_group_create_job(synch_count, host_objects, metahost_objects, dependencies, atomic_group, labels_by_name): """ Attempt to reject create_job requests with an atomic group that will be impossible to schedule. The checks are not perfect but should catch the most obvious issues. @param synch_count - The job's minimum synch count. @param host_objects - A list of models.Host instances. @param metahost_objects - A list of models.Label instances. @param dependencies - A list of job dependency label names. @param atomic_group - The models.AtomicGroup instance. @param labels_by_name - A dictionary mapping label names to models.Label instance. Used to look up instances for dependencies. @raises model_logic.ValidationError - When an issue is found. """ # If specific host objects were supplied with an atomic group, verify # that there are enough to satisfy the synch_count. minimum_required = synch_count or 1 if (host_objects and not metahost_objects and len(host_objects) < minimum_required): raise model_logic.ValidationError({ 'hosts': 'only %d hosts provided for job with synch_count = %d' % (len(host_objects), synch_count) }) # Check that the atomic group has a hope of running this job # given any supplied metahosts and dependancies that may limit. # Get a set of hostnames in the atomic group. possible_hosts = set() for label in atomic_group.label_set.all(): possible_hosts.update(h.hostname for h in label.host_set.all()) # Filter out hosts that don't match all of the job dependency labels. for label_name in set(dependencies): label = labels_by_name[label_name] hosts_in_label = (h.hostname for h in label.host_set.all()) possible_hosts.intersection_update(hosts_in_label) if not host_objects and not metahost_objects: # No hosts or metahosts are required to queue an atomic group Job. # However, if they are given, we respect them below. host_set = possible_hosts else: host_set = set(host.hostname for host in host_objects) unusable_host_set = host_set.difference(possible_hosts) if unusable_host_set: raise model_logic.ValidationError({ 'hosts': 'Hosts "%s" are not in Atomic Group "%s"' % (', '.join(sorted(unusable_host_set)), atomic_group.name) }) # Lookup hosts provided by each meta host and merge them into the # host_set for final counting. for meta_host in metahost_objects: meta_possible = possible_hosts.copy() hosts_in_meta_host = (h.hostname for h in meta_host.host_set.all()) meta_possible.intersection_update(hosts_in_meta_host) # Count all hosts that this meta_host will provide. host_set.update(meta_possible) if len(host_set) < minimum_required: raise model_logic.ValidationError({ 'atomic_group_name': 'Insufficient hosts in Atomic Group "%s" with the' ' supplied dependencies and meta_hosts.' % (atomic_group.name, ) })