class LaunchTimeConfig(LaunchTimeConfigBase): ''' Common model for all objects that save details of a saved launch config WFJT / WJ nodes, schedules, and job launch configs (not all implemented yet) ''' class Meta: abstract = True # Special case prompting fields, even more special than the other ones extra_data = JSONField(blank=True, default={}) survey_passwords = prevent_search( JSONField( blank=True, default={}, editable=False, )) # Credentials needed for non-unified job / unified JT models credentials = models.ManyToManyField('Credential', related_name='%(class)ss') @property def extra_vars(self): return self.extra_data @extra_vars.setter def extra_vars(self, extra_vars): self.extra_data = extra_vars
class LaunchTimeConfig(LaunchTimeConfigBase): ''' Common model for all objects that save details of a saved launch config WFJT / WJ nodes, schedules, and job launch configs (not all implemented yet) ''' class Meta: abstract = True # Special case prompting fields, even more special than the other ones extra_data = JSONField( blank=True, default=dict ) survey_passwords = prevent_search(JSONField( blank=True, default=dict, editable=False, )) # Credentials needed for non-unified job / unified JT models credentials = models.ManyToManyField( 'Credential', related_name='%(class)ss' ) @property def extra_vars(self): return self.extra_data @extra_vars.setter def extra_vars(self, extra_vars): self.extra_data = extra_vars def display_extra_vars(self): ''' Hides fields marked as passwords in survey. ''' if hasattr(self, 'survey_passwords') and self.survey_passwords: extra_vars = parse_yaml_or_json(self.extra_vars).copy() for key, value in self.survey_passwords.items(): if key in extra_vars: extra_vars[key] = value return extra_vars else: return self.extra_vars def display_extra_data(self): return self.display_extra_vars()
class LaunchTimeConfigBase(BaseModel): ''' Needed as separate class from LaunchTimeConfig because some models use `extra_data` and some use `extra_vars`. We cannot change the API, so we force fake it in the model definitions - model defines extra_vars - use this class - model needs to use extra data - use LaunchTimeConfig Use this for models which are SurveyMixins and UnifiedJobs or Templates ''' class Meta: abstract = True # Prompting-related fields that have to be handled as special cases inventory = models.ForeignKey( 'Inventory', related_name='%(class)ss', blank=True, null=True, default=None, on_delete=models.SET_NULL, help_text=_('Inventory applied as a prompt, assuming job template prompts for inventory') ) # All standard fields are stored in this dictionary field # This is a solution to the nullable CharField problem, specific to prompting char_prompts = JSONField( blank=True, default=dict ) def prompts_dict(self, display=False): data = {} # Some types may have different prompts, but always subset of JT prompts for prompt_name in JobTemplate.get_ask_mapping().keys(): try: field = self._meta.get_field(prompt_name) except FieldDoesNotExist: field = None if isinstance(field, models.ManyToManyField): if not self.pk: continue # unsaved object can't have related many-to-many prompt_val = set(getattr(self, prompt_name).all()) if len(prompt_val) > 0: data[prompt_name] = prompt_val elif prompt_name == 'extra_vars': if self.extra_vars: if display: data[prompt_name] = self.display_extra_vars() else: data[prompt_name] = self.extra_vars # Depending on model, field type may save and return as string if isinstance(data[prompt_name], str): data[prompt_name] = parse_yaml_or_json(data[prompt_name]) if self.survey_passwords and not display: data['survey_passwords'] = self.survey_passwords else: prompt_val = getattr(self, prompt_name) if prompt_val is not None: data[prompt_name] = prompt_val return data
class Setting(CreatedModifiedModel): key = models.CharField(max_length=255, ) value = JSONField(null=True, ) user = prevent_search( models.ForeignKey( 'auth.User', related_name='settings', default=None, null=True, editable=False, on_delete=models.CASCADE, )) def __str__(self): try: json_value = json.dumps(self.value) except ValueError: # In the rare case the DB value is invalid JSON. json_value = u'<Invalid JSON>' if self.user: return u'{} ({}) = {}'.format(self.key, self.user, json_value) else: return u'{} = {}'.format(self.key, json_value) def save(self, *args, **kwargs): encrypted = settings_registry.is_setting_encrypted(self.key) new_instance = not bool(self.pk) # If update_fields has been specified, add our field names to it, # if it hasn't been specified, then we're just doing a normal save. update_fields = kwargs.get('update_fields', []) # When first saving to the database, don't store any encrypted field # value, but instead save it until after the instance is created. # Otherwise, store encrypted value to the database. if encrypted: if new_instance: self._saved_value = self.value self.value = '' else: self.value = encrypt_field(self, 'value') if 'value' not in update_fields: update_fields.append('value') super(Setting, self).save(*args, **kwargs) # After saving a new instance for the first time, set the encrypted # field and save again. if encrypted and new_instance: from awx.main.signals import disable_activity_stream with disable_activity_stream(): self.value = self._saved_value self.save(update_fields=['value']) @classmethod def get_cache_key(self, key): return key @classmethod def get_cache_id_key(self, key): return '{}_ID'.format(key)
class Notification(CreatedModifiedModel): ''' A notification event emitted when a NotificationTemplate is run ''' NOTIFICATION_STATE_CHOICES = [ ('pending', _('Pending')), ('successful', _('Successful')), ('failed', _('Failed')), ] class Meta: app_label = 'main' ordering = ('pk', ) notification_template = models.ForeignKey('NotificationTemplate', related_name='notifications', on_delete=models.CASCADE, editable=False) status = models.CharField( max_length=20, choices=NOTIFICATION_STATE_CHOICES, default='pending', editable=False, ) error = models.TextField( blank=True, default='', editable=False, ) notifications_sent = models.IntegerField( default=0, editable=False, ) notification_type = models.CharField( max_length=32, choices=NotificationTemplate.NOTIFICATION_TYPE_CHOICES, ) recipients = models.TextField( blank=True, default='', editable=False, ) subject = models.TextField( blank=True, default='', editable=False, ) body = JSONField(blank=True) def get_absolute_url(self, request=None): return reverse('api:notification_detail', kwargs={'pk': self.pk}, request=request)
class InstanceGroup(models.Model, RelatedJobsMixin): """A model representing a Queue/Group of AWX Instances.""" objects = InstanceGroupManager() name = models.CharField(max_length=250, unique=True) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) instances = models.ManyToManyField( 'Instance', related_name='rampart_groups', editable=False, help_text=_('Instances that are members of this InstanceGroup'), ) controller = models.ForeignKey( 'InstanceGroup', related_name='controlled_groups', help_text=_('Instance Group to remotely control this group.'), editable=False, default=None, null=True) policy_instance_percentage = models.IntegerField( default=0, help_text=_( "Percentage of Instances to automatically assign to this group")) policy_instance_minimum = models.IntegerField( default=0, help_text= _("Static minimum number of Instances to automatically assign to this group" )) policy_instance_list = JSONField( default=[], blank=True, help_text= _("List of exact-match Instances that will always be automatically assigned to this group" )) def get_absolute_url(self, request=None): return reverse('api:instance_group_detail', kwargs={'pk': self.pk}, request=request) @property def capacity(self): return sum([inst.capacity for inst in self.instances.all()]) ''' RelatedJobsMixin ''' def _get_related_jobs(self): return UnifiedJob.objects.filter(instance_group=self) class Meta: app_label = 'main'
class SurveyJobMixin(models.Model): class Meta: abstract = True survey_passwords = prevent_search( JSONField( blank=True, default={}, editable=False, )) def display_extra_vars(self): ''' Hides fields marked as passwords in survey. ''' if self.survey_passwords: extra_vars = json.loads(self.extra_vars) for key, value in self.survey_passwords.items(): if key in extra_vars: extra_vars[key] = value return json.dumps(extra_vars) else: return self.extra_vars def decrypted_extra_vars(self): ''' Decrypts fields marked as passwords in survey. ''' if self.survey_passwords: extra_vars = json.loads(self.extra_vars) for key in self.survey_passwords: value = extra_vars.get(key) if value and isinstance( value, six.string_types) and value.startswith('$encrypted$'): extra_vars[key] = decrypt_value( get_encryption_key('value', pk=None), value) return json.dumps(extra_vars) else: return self.extra_vars
class SurveyJobMixin(models.Model): class Meta: abstract = True survey_passwords = prevent_search( JSONField( blank=True, default={}, editable=False, )) def display_extra_vars(self): ''' Hides fields marked as passwords in survey. ''' if self.survey_passwords: extra_vars = json.loads(self.extra_vars) for key, value in self.survey_passwords.items(): if key in extra_vars: extra_vars[key] = value return json.dumps(extra_vars) else: return self.extra_vars
class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin): ''' A job applies a project (with playbook) to an inventory source with a given credential. It represents a single invocation of ansible-playbook with the given parameters. ''' class Meta: app_label = 'main' ordering = ('id', ) job_template = models.ForeignKey( 'JobTemplate', related_name='jobs', blank=True, null=True, default=None, on_delete=models.SET_NULL, ) hosts = models.ManyToManyField( 'Host', related_name='jobs', editable=False, through='JobHostSummary', ) artifacts = JSONField( blank=True, default={}, editable=False, ) scm_revision = models.CharField( max_length=1024, blank=True, default='', editable=False, verbose_name=_('SCM Revision'), help_text=_( 'The SCM Revision from the Project used for this job, if available' ), ) project_update = models.ForeignKey( 'ProjectUpdate', blank=True, null=True, default=None, on_delete=models.SET_NULL, help_text= _('The SCM Refresh task used to make sure the playbooks were available for the job run' ), ) @classmethod def _get_parent_field_name(cls): return 'job_template' @classmethod def _get_task_class(cls): from awx.main.tasks import RunJob return RunJob @classmethod def supports_isolation(cls): return True def _global_timeout_setting(self): return 'DEFAULT_JOB_TIMEOUT' @classmethod def _get_unified_job_template_class(cls): return JobTemplate def get_absolute_url(self, request=None): return reverse('api:job_detail', kwargs={'pk': self.pk}, request=request) def get_ui_url(self): return urljoin(settings.TOWER_URL_BASE, "/#/jobs/{}".format(self.pk)) @property def ansible_virtualenv_path(self): # the order here enforces precedence (it matters) for virtualenv in (self.job_template.custom_virtualenv if self.job_template else None, self.project.custom_virtualenv, self.project.organization.custom_virtualenv): if virtualenv: return virtualenv return settings.ANSIBLE_VENV_PATH @property def event_class(self): return JobEvent @property def ask_diff_mode_on_launch(self): if self.job_template is not None: return self.job_template.ask_diff_mode_on_launch return False @property def ask_variables_on_launch(self): if self.job_template is not None: return self.job_template.ask_variables_on_launch return False @property def ask_limit_on_launch(self): if self.job_template is not None: return self.job_template.ask_limit_on_launch return False @property def ask_tags_on_launch(self): if self.job_template is not None: return self.job_template.ask_tags_on_launch return False @property def ask_skip_tags_on_launch(self): if self.job_template is not None: return self.job_template.ask_skip_tags_on_launch return False @property def ask_job_type_on_launch(self): if self.job_template is not None: return self.job_template.ask_job_type_on_launch return False @property def ask_verbosity_on_launch(self): if self.job_template is not None: return self.job_template.ask_verbosity_on_launch return False @property def ask_inventory_on_launch(self): if self.job_template is not None: return self.job_template.ask_inventory_on_launch return False @property def ask_credential_on_launch(self): if self.job_template is not None: return self.job_template.ask_credential_on_launch return False def get_passwords_needed_to_start(self): return self.passwords_needed_to_start def _get_hosts(self, **kwargs): Host = JobHostSummary._meta.get_field('host').related_model kwargs['job_host_summaries__job__pk'] = self.pk return Host.objects.filter(**kwargs) def retry_qs(self, status): ''' Returns Host queryset that will be used to produce the `limit` field in a retry on a subset of hosts ''' kwargs = {} if status == 'all': pass elif status == 'failed': # Special case for parity with Ansible .retry files kwargs['job_host_summaries__failed'] = True elif status in ['ok', 'changed', 'unreachable']: if status == 'unreachable': status_field = 'dark' else: status_field = status kwargs['job_host_summaries__{}__gt'.format(status_field)] = 0 else: raise ParseError( _('{status_value} is not a valid status option.').format( status_value=status)) return self._get_hosts(**kwargs) @property def task_impact(self): # NOTE: We sorta have to assume the host count matches and that forks default to 5 from awx.main.models.inventory import Host if self.launch_type == 'callback': count_hosts = 1 else: count_hosts = Host.objects.filter( inventory__jobs__pk=self.pk).count() return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1 @property def successful_hosts(self): return self._get_hosts(job_host_summaries__ok__gt=0) @property def failed_hosts(self): return self._get_hosts(job_host_summaries__failures__gt=0) @property def changed_hosts(self): return self._get_hosts(job_host_summaries__changed__gt=0) @property def dark_hosts(self): return self._get_hosts(job_host_summaries__dark__gt=0) @property def unreachable_hosts(self): return self.dark_hosts @property def skipped_hosts(self): return self._get_hosts(job_host_summaries__skipped__gt=0) @property def processed_hosts(self): return self._get_hosts(job_host_summaries__processed__gt=0) def notification_data(self, block=5): data = super(Job, self).notification_data() all_hosts = {} # NOTE: Probably related to job event slowness, remove at some point -matburt if block: summaries = self.job_host_summaries.all() while block > 0 and not len(summaries): time.sleep(1) block -= 1 else: summaries = self.job_host_summaries.all() for h in self.job_host_summaries.all(): all_hosts[h.host_name] = dict(failed=h.failed, changed=h.changed, dark=h.dark, failures=h.failures, ok=h.ok, processed=h.processed, skipped=h.skipped) data.update( dict(inventory=self.inventory.name if self.inventory else None, project=self.project.name if self.project else None, playbook=self.playbook, credential=getattr(self.get_deprecated_credential('ssh'), 'name', None), limit=self.limit, extra_vars=self.display_extra_vars(), hosts=all_hosts)) return data def _resources_sufficient_for_launch(self): return not (self.inventory_id is None or self.project_id is None) def display_artifacts(self): ''' Hides artifacts if they are marked as no_log type artifacts. ''' artifacts = self.artifacts if artifacts.get('_ansible_no_log', False): return "$hidden due to Ansible no_log flag$" return artifacts @property def preferred_instance_groups(self): if self.project is not None and self.project.organization is not None: organization_groups = [ x for x in self.project.organization.instance_groups.all() ] else: organization_groups = [] if self.inventory is not None: inventory_groups = [ x for x in self.inventory.instance_groups.all() ] else: inventory_groups = [] if self.job_template is not None: template_groups = [ x for x in self.job_template.instance_groups.all() ] else: template_groups = [] selected_groups = template_groups + inventory_groups + organization_groups if not selected_groups: return self.global_instance_groups return selected_groups def awx_meta_vars(self): r = super(Job, self).awx_meta_vars() if self.project: for name in ('awx', 'tower'): r['{}_project_revision'.format( name)] = self.project.scm_revision if self.job_template: for name in ('awx', 'tower'): r['{}_job_template_id'.format(name)] = self.job_template.pk r['{}_job_template_name'.format(name)] = self.job_template.name return r ''' JobNotificationMixin ''' def get_notification_templates(self): if not self.job_template: return NotificationTemplate.objects.none() return self.job_template.notification_templates def get_notification_friendly_name(self): return "Job" @property def memcached_fact_key(self): return '{}'.format(self.inventory.id) def memcached_fact_host_key(self, host_name): return '{}-{}'.format(self.inventory.id, base64.b64encode(host_name.encode('utf-8'))) def memcached_fact_modified_key(self, host_name): return '{}-{}-modified'.format( self.inventory.id, base64.b64encode(host_name.encode('utf-8'))) def _get_inventory_hosts(self, only=[ 'name', 'ansible_facts', 'modified', ]): return self.inventory.hosts.only(*only) def _get_memcache_connection(self): return memcache.Client([settings.CACHES['default']['LOCATION']], debug=0) def start_job_fact_cache(self): if not self.inventory: return cache = self._get_memcache_connection() host_names = [] for host in self._get_inventory_hosts(): host_key = self.memcached_fact_host_key(host.name) modified_key = self.memcached_fact_modified_key(host.name) if cache.get(modified_key) is None: if host.ansible_facts_modified: host_modified = host.ansible_facts_modified.replace( tzinfo=tzutc()).isoformat() else: host_modified = datetime.datetime.now(tzutc()).isoformat() cache.set(host_key, json.dumps(host.ansible_facts)) cache.set(modified_key, host_modified) host_names.append(host.name) cache.set(self.memcached_fact_key, host_names) def finish_job_fact_cache(self): if not self.inventory: return cache = self._get_memcache_connection() hosts = self._get_inventory_hosts() for host in hosts: host_key = self.memcached_fact_host_key(host.name) modified_key = self.memcached_fact_modified_key(host.name) modified = cache.get(modified_key) if modified is None: cache.delete(host_key) continue # Save facts if cache is newer than DB modified = parser.parse(modified, tzinfos=[tzutc()]) if not host.ansible_facts_modified or modified > host.ansible_facts_modified: ansible_facts = cache.get(host_key) try: ansible_facts = json.loads(ansible_facts) except Exception: ansible_facts = None if ansible_facts is None: cache.delete(host_key) continue host.ansible_facts = ansible_facts host.ansible_facts_modified = modified if 'insights' in ansible_facts and 'system_id' in ansible_facts[ 'insights']: host.insights_system_id = ansible_facts['insights'][ 'system_id'] host.save() system_tracking_logger.info( 'New fact for inventory {} host {}'.format( smart_str(host.inventory.name), smart_str(host.name)), extra=dict(inventory_id=host.inventory.id, host_name=host.name, ansible_facts=host.ansible_facts, ansible_facts_modified=host. ansible_facts_modified.isoformat(), job_id=self.id))
class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique, UnifiedJobTypeStringMixin, TaskManagerUnifiedJobMixin): ''' Concrete base class for unified job run by the task engine. ''' STATUS_CHOICES = UnifiedJobTemplate.JOB_STATUS_CHOICES LAUNCH_TYPE_CHOICES = [ ('manual', _('Manual')), # Job was started manually by a user. ('relaunch', _('Relaunch')), # Job was started via relaunch. ('callback', _('Callback')), # Job was started via host callback. ('scheduled', _('Scheduled')), # Job was started from a schedule. ('dependency', _('Dependency')), # Job was started as a dependency of another job. ('workflow', _('Workflow')), # Job was started from a workflow job. ('sync', _('Sync')), # Job was started from a project sync. ('scm', _('SCM Update')) # Job was created as an Inventory SCM sync. ] PASSWORD_FIELDS = ('start_args', ) # NOTE: Working around a django-polymorphic issue: https://github.com/django-polymorphic/django-polymorphic/issues/229 base_manager_name = 'base_objects' class Meta: app_label = 'main' old_pk = models.PositiveIntegerField( null=True, default=None, editable=False, ) unified_job_template = models.ForeignKey( 'UnifiedJobTemplate', null=True, # Some jobs can be run without a template. default=None, editable=False, related_name='%(class)s_unified_jobs', on_delete=models.SET_NULL, ) launch_type = models.CharField( max_length=20, choices=LAUNCH_TYPE_CHOICES, default='manual', editable=False, ) schedule = models.ForeignKey( # Which schedule entry was responsible for starting this job. 'Schedule', null=True, default=None, editable=False, on_delete=models.SET_NULL, ) dependent_jobs = models.ManyToManyField( 'self', editable=False, related_name='%(class)s_blocked_jobs+', ) execution_node = models.TextField( blank=True, default='', editable=False, help_text=_("The node the job executed on."), ) notifications = models.ManyToManyField( 'Notification', editable=False, related_name='%(class)s_notifications', ) cancel_flag = models.BooleanField( blank=True, default=False, editable=False, ) status = models.CharField( max_length=20, choices=STATUS_CHOICES, default='new', editable=False, ) failed = models.BooleanField( default=False, editable=False, ) started = models.DateTimeField( null=True, default=None, editable=False, help_text=_("The date and time the job was queued for starting."), ) finished = models.DateTimeField( null=True, default=None, editable=False, help_text=_("The date and time the job finished execution."), ) elapsed = models.DecimalField( max_digits=12, decimal_places=3, editable=False, help_text=_("Elapsed time in seconds that the job ran."), ) job_args = prevent_search( models.TextField( blank=True, default='', editable=False, )) job_cwd = models.CharField( max_length=1024, blank=True, default='', editable=False, ) job_env = prevent_search( JSONField( blank=True, default={}, editable=False, )) job_explanation = models.TextField( blank=True, default='', editable=False, help_text= _("A status field to indicate the state of the job if it wasn't able to run and capture stdout" ), ) start_args = prevent_search( models.TextField( blank=True, default='', editable=False, )) result_stdout_text = models.TextField( blank=True, default='', editable=False, ) result_stdout_file = models.TextField( # FilePathfield? blank=True, default='', editable=False, ) result_traceback = models.TextField( blank=True, default='', editable=False, ) celery_task_id = models.CharField( max_length=100, blank=True, default='', editable=False, ) labels = models.ManyToManyField("Label", blank=True, related_name='%(class)s_labels') instance_group = models.ForeignKey( 'InstanceGroup', blank=True, null=True, default=None, on_delete=models.SET_NULL, help_text=_('The Rampart/Instance group the job was run under'), ) credentials = models.ManyToManyField( 'Credential', related_name='%(class)ss', ) def get_absolute_url(self, request=None): real_instance = self.get_real_instance() if real_instance != self: return real_instance.get_absolute_url(request=request) else: return '' def get_ui_url(self): real_instance = self.get_real_instance() if real_instance != self: return real_instance.get_ui_url() else: return '' @classmethod def _get_task_class(cls): raise NotImplementedError # Implement in subclasses. @classmethod def supports_isolation(cls): return False @classmethod def _get_parent_field_name(cls): return 'unified_job_template' # Override in subclasses. @classmethod def _get_unified_job_template_class(cls): ''' Return subclass of UnifiedJobTemplate that applies to this unified job. ''' raise NotImplementedError # Implement in subclass. def _global_timeout_setting(self): "Override in child classes, None value indicates this is not configurable" return None def _resources_sufficient_for_launch(self): return True def __unicode__(self): return u'%s-%s-%s' % (self.created, self.id, self.status) @property def log_format(self): return '{} {} ({})'.format(get_type_for_model(type(self)), self.id, self.status) def _get_parent_instance(self): return getattr(self, self._get_parent_field_name(), None) def _update_parent_instance_no_save(self, parent_instance, update_fields=[]): def parent_instance_set(key, val): setattr(parent_instance, key, val) if key not in update_fields: update_fields.append(key) if parent_instance: if self.status in ('pending', 'waiting', 'running'): if parent_instance.current_job != self: parent_instance_set('current_job', self) # Update parent with all the 'good' states of it's child if parent_instance.status != self.status: parent_instance_set('status', self.status) elif self.status in ('successful', 'failed', 'error', 'canceled'): if parent_instance.current_job == self: parent_instance_set('current_job', None) parent_instance_set('last_job', self) parent_instance_set('last_job_failed', self.failed) return update_fields def _update_parent_instance(self): parent_instance = self._get_parent_instance() if parent_instance: update_fields = self._update_parent_instance_no_save( parent_instance) parent_instance.save(update_fields=update_fields) def save(self, *args, **kwargs): """Save the job, with current status, to the database. Ensure that all data is consistent before doing so. """ # If update_fields has been specified, add our field names to it, # if it hasn't been specified, then we're just doing a normal save. update_fields = kwargs.get('update_fields', []) # Get status before save... status_before = self.status or 'new' # If this job already exists in the database, retrieve a copy of # the job in its prior state. if self.pk: self_before = self.__class__.objects.get(pk=self.pk) if self_before.status != self.status: status_before = self_before.status # Sanity check: Is this a failure? Ensure that the failure value # matches the status. failed = bool(self.status in ('failed', 'error', 'canceled')) if self.failed != failed: self.failed = failed if 'failed' not in update_fields: update_fields.append('failed') # Sanity check: Has the job just started? If so, mark down its start # time. if self.status == 'running' and not self.started: self.started = now() if 'started' not in update_fields: update_fields.append('started') # Sanity check: Has the job just completed? If so, mark down its # completion time, and record its output to the database. if self.status in ('successful', 'failed', 'error', 'canceled') and not self.finished: # Record the `finished` time. self.finished = now() if 'finished' not in update_fields: update_fields.append('finished') # If we have a start and finished time, and haven't already calculated # out the time that elapsed, do so. if self.started and self.finished and not self.elapsed: td = self.finished - self.started elapsed = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / (10**6 * 1.0) else: elapsed = 0.0 if self.elapsed != elapsed: self.elapsed = str(elapsed) if 'elapsed' not in update_fields: update_fields.append('elapsed') # Ensure that the job template information is current. if self.unified_job_template != self._get_parent_instance(): self.unified_job_template = self._get_parent_instance() if 'unified_job_template' not in update_fields: update_fields.append('unified_job_template') # Okay; we're done. Perform the actual save. result = super(UnifiedJob, self).save(*args, **kwargs) # If status changed, update the parent instance. if self.status != status_before: self._update_parent_instance() # Done. return result def delete(self): if self.result_stdout_file != "": try: os.remove(self.result_stdout_file) except Exception: pass super(UnifiedJob, self).delete() def copy_unified_job(self, limit=None): ''' Returns saved object, including related fields. Create a copy of this unified job for the purpose of relaunch ''' unified_job_class = self.__class__ unified_jt_class = self._get_unified_job_template_class() parent_field_name = unified_job_class._get_parent_field_name() fields = unified_jt_class._get_unified_job_field_names() + [ parent_field_name ] unified_job = copy_model_by_class(self, unified_job_class, fields, {}) unified_job.launch_type = 'relaunch' if limit: unified_job.limit = limit unified_job.save() # Labels coppied here copy_m2m_relationships(self, unified_job, fields) return unified_job def result_stdout_raw_handle(self, attempt=0): """Return a file-like object containing the standard out of the job's result. """ msg = { 'pending': 'Waiting for results...', 'missing': 'stdout capture is missing', } if self.result_stdout_text: return StringIO(self.result_stdout_text) else: if not os.path.exists(self.result_stdout_file) or os.stat( self.result_stdout_file).st_size < 1: return StringIO(msg['missing' if self.finished else 'pending']) # There is a potential timing issue here, because another # process may be deleting the stdout file after it is written # to the database. # # Therefore, if we get an IOError (which generally means the # file does not exist), reload info from the database and # try again. try: return codecs.open(self.result_stdout_file, "r", encoding='utf-8') except IOError: if attempt < 3: self.result_stdout_text = type(self).objects.get( id=self.id).result_stdout_text return self.result_stdout_raw_handle(attempt=attempt + 1) else: return StringIO( msg['missing' if self.finished else 'pending']) def _escape_ascii(self, content): # Remove ANSI escape sequences used to embed event data. content = re.sub(r'\x1b\[K(?:[A-Za-z0-9+/=]+\x1b\[\d+D)+\x1b\[K', '', content) # Remove ANSI color escape sequences. content = re.sub(r'\x1b[^m]*m', '', content) return content def _result_stdout_raw(self, redact_sensitive=False, escape_ascii=False): content = self.result_stdout_raw_handle().read() if redact_sensitive: content = UriCleaner.remove_sensitive(content) if escape_ascii: content = self._escape_ascii(content) return content @property def result_stdout_raw(self): return self._result_stdout_raw() @property def result_stdout(self): return self._result_stdout_raw(escape_ascii=True) @property def result_stdout_size(self): try: return os.stat(self.result_stdout_file).st_size except Exception: return len(self.result_stdout) def _result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True, escape_ascii=False): return_buffer = u"" if end_line is not None: end_line = int(end_line) stdout_lines = self.result_stdout_raw_handle().readlines() absolute_end = len(stdout_lines) for line in stdout_lines[int(start_line):end_line]: return_buffer += line if int(start_line) < 0: start_actual = len(stdout_lines) + int(start_line) end_actual = len(stdout_lines) else: start_actual = int(start_line) if end_line is not None: end_actual = min(int(end_line), len(stdout_lines)) else: end_actual = len(stdout_lines) if redact_sensitive: return_buffer = UriCleaner.remove_sensitive(return_buffer) if escape_ascii: return_buffer = self._escape_ascii(return_buffer) return return_buffer, start_actual, end_actual, absolute_end def result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=False): return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive) def result_stdout_limited(self, start_line=0, end_line=None, redact_sensitive=False): return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive, escape_ascii=True) @property def spawned_by_workflow(self): return self.launch_type == 'workflow' @property def workflow_job_id(self): if self.spawned_by_workflow: try: return self.unified_job_node.workflow_job.pk except UnifiedJob.unified_job_node.RelatedObjectDoesNotExist: pass return None @property def workflow_node_id(self): if self.spawned_by_workflow: try: return self.unified_job_node.pk except UnifiedJob.unified_job_node.RelatedObjectDoesNotExist: pass return None @property def celery_task(self): try: if self.celery_task_id: return TaskResult.objects.get(task_id=self.celery_task_id) except TaskResult.DoesNotExist: pass def get_passwords_needed_to_start(self): return [] def handle_extra_data(self, extra_data): if hasattr(self, 'extra_vars') and extra_data: extra_data_dict = {} try: extra_data_dict = parse_yaml_or_json(extra_data, silent_failure=False) except Exception as e: logger.warn("Exception deserializing extra vars: " + str(e)) evars = self.extra_vars_dict evars.update(extra_data_dict) self.update_fields(extra_vars=json.dumps(evars)) @property def can_start(self): return bool(self.status in ('new', 'waiting')) @property def task_impact(self): raise NotImplementedError # Implement in subclass. def websocket_emit_data(self): ''' Return extra data that should be included when submitting data to the browser over the websocket connection ''' websocket_data = dict() if self.spawned_by_workflow: websocket_data.update( dict(workflow_job_id=self.workflow_job_id, workflow_node_id=self.workflow_node_id)) return websocket_data def _websocket_emit_status(self, status): try: status_data = dict(unified_job_id=self.id, status=status) if status == 'waiting': if self.instance_group: status_data[ 'instance_group_name'] = self.instance_group.name else: status_data['instance_group_name'] = None status_data.update(self.websocket_emit_data()) status_data['group_name'] = 'jobs' emit_channel_notification('jobs-status_changed', status_data) if self.spawned_by_workflow: status_data['group_name'] = "workflow_events" emit_channel_notification( 'workflow_events-' + str(self.workflow_job_id), status_data) except IOError: # includes socket errors logger.exception( '%s failed to emit channel msg about status change', self.log_format) def websocket_emit_status(self, status): connection.on_commit(lambda: self._websocket_emit_status(status)) def notification_data(self): return dict(id=self.id, name=self.name, url=self.get_ui_url(), created_by=smart_text(self.created_by), started=self.started.isoformat() if self.started is not None else None, finished=self.finished.isoformat() if self.finished is not None else None, status=self.status, traceback=self.result_traceback) def pre_start(self, **kwargs): if not self.can_start: self.job_explanation = u'%s is not in a startable state: %s, expecting one of %s' % ( self._meta.verbose_name, self.status, str(('new', 'waiting'))) self.save(update_fields=['job_explanation']) return (False, None) needed = self.get_passwords_needed_to_start() try: start_args = json.loads(decrypt_field(self, 'start_args')) except Exception: start_args = None if start_args in (None, ''): start_args = kwargs opts = dict([(field, start_args.get(field, '')) for field in needed]) if not all(opts.values()): missing_fields = ', '.join([k for k, v in opts.items() if not v]) self.job_explanation = u'Missing needed fields: %s.' % missing_fields self.save(update_fields=['job_explanation']) return (False, None) if 'extra_vars' in kwargs: self.handle_extra_data(kwargs['extra_vars']) return (True, opts) def start_celery_task(self, opts, error_callback, success_callback, queue): kwargs = { 'link_error': error_callback, 'link': success_callback, 'queue': None, 'task_id': None, } if not self.celery_task_id: raise RuntimeError("Expected celery_task_id to be set on model.") kwargs['task_id'] = self.celery_task_id task_class = self._get_task_class() from awx.main.models.ha import InstanceGroup ig = InstanceGroup.objects.get(name=queue) args = [self.pk] if ig.controller_id: if self.supports_isolation(): # case of jobs and ad hoc commands isolated_instance = ig.instances.order_by('-capacity').first() args.append(isolated_instance.hostname) else: # proj & inv updates, system jobs run on controller queue = ig.controller.name kwargs['queue'] = queue task_class().apply_async(args, opts, **kwargs) def start(self, error_callback, success_callback, **kwargs): ''' Start the task running via Celery. ''' (res, opts) = self.pre_start(**kwargs) if res: self.start_celery_task(opts, error_callback, success_callback) return res def signal_start(self, **kwargs): """Notify the task runner system to begin work on this task.""" # Sanity check: Are we able to start the job? If not, do not attempt # to do so. if not self.can_start: return False # Get any passwords or other data that are prerequisites to running # the job. needed = self.get_passwords_needed_to_start() opts = dict([(field, kwargs.get(field, '')) for field in needed]) if not all(opts.values()): return False if 'extra_vars' in kwargs: self.handle_extra_data(kwargs['extra_vars']) # Sanity check: If we are running unit tests, then run synchronously. if getattr(settings, 'CELERY_UNIT_TEST', False): return self.start(None, None, **kwargs) # Save the pending status, and inform the SocketIO listener. self.update_fields(start_args=json.dumps(kwargs), status='pending') self.websocket_emit_status("pending") from awx.main.scheduler.tasks import run_job_launch connection.on_commit(lambda: run_job_launch.delay(self.id)) # Each type of unified job has a different Task class; get the # appropirate one. # task_type = get_type_for_model(self) # Actually tell the task runner to run this task. # FIXME: This will deadlock the task runner #from awx.main.tasks import notify_task_runner #notify_task_runner.delay({'id': self.id, 'metadata': kwargs, # 'task_type': task_type}) # Done! return True @property def can_cancel(self): return bool(self.status in CAN_CANCEL) def _force_cancel(self): # Update the status to 'canceled' if we can detect that the job # really isn't running (i.e. celery has crashed or forcefully # killed the worker). task_statuses = ('STARTED', 'SUCCESS', 'FAILED', 'RETRY', 'REVOKED') try: taskmeta = self.celery_task if not taskmeta or taskmeta.status not in task_statuses: return from celery import current_app i = current_app.control.inspect() for v in (i.active() or {}).values(): if taskmeta.task_id in [x['id'] for x in v]: return for v in (i.reserved() or {}).values(): if taskmeta.task_id in [x['id'] for x in v]: return for v in (i.revoked() or {}).values(): if taskmeta.task_id in [x['id'] for x in v]: return for v in (i.scheduled() or {}).values(): if taskmeta.task_id in [x['id'] for x in v]: return instance = self.__class__.objects.get(pk=self.pk) if instance.can_cancel: instance.status = 'canceled' update_fields = ['status'] if not instance.job_explanation: instance.job_explanation = 'Forced cancel' update_fields.append('job_explanation') instance.save(update_fields=update_fields) self.websocket_emit_status("canceled") except Exception: # FIXME: Log this exception! if settings.DEBUG: raise def _build_job_explanation(self): if not self.job_explanation: return 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \ (self.model_to_str(), self.name, self.id) return None def cancel(self, job_explanation=None, is_chain=False): if self.can_cancel: if not is_chain: map( lambda x: x.cancel(job_explanation=self. _build_job_explanation(), is_chain=True), self.get_jobs_fail_chain()) if not self.cancel_flag: self.cancel_flag = True cancel_fields = ['cancel_flag'] if self.status in ('pending', 'waiting', 'new'): self.status = 'canceled' cancel_fields.append('status') if job_explanation is not None: self.job_explanation = job_explanation cancel_fields.append('job_explanation') self.save(update_fields=cancel_fields) self.websocket_emit_status("canceled") if settings.CELERY_BROKER_URL.startswith('amqp://'): self._force_cancel() return self.cancel_flag @property def preferred_instance_groups(self): ''' Return Instance/Rampart Groups preferred by this unified job templates ''' if not self.unified_job_template: return [] template_groups = [ x for x in self.unified_job_template.instance_groups.all() ] return template_groups @property def global_instance_groups(self): from awx.main.models.ha import InstanceGroup default_instance_group = InstanceGroup.objects.filter(name='tower') if default_instance_group.exists(): return [default_instance_group.first()] return []
class WorkflowJobNode(WorkflowNodeBase): job = models.OneToOneField( 'UnifiedJob', related_name='unified_job_node', blank=True, null=True, default=None, on_delete=models.SET_NULL, ) workflow_job = models.ForeignKey( 'WorkflowJob', related_name='workflow_job_nodes', blank=True, null=True, default=None, on_delete=models.CASCADE, ) ancestor_artifacts = JSONField( blank=True, default={}, editable=False, ) def get_absolute_url(self, request=None): return reverse('api:workflow_job_node_detail', kwargs={'pk': self.pk}, request=request) def get_job_kwargs(self): ''' In advance of creating a new unified job as part of a workflow, this method builds the attributes to use It alters the node by saving its updated version of ancestor_artifacts, making it available to subsequent nodes. ''' # reject/accept prompted fields data = {} ujt_obj = self.unified_job_template if ujt_obj is not None: accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs(**self.prompts_dict()) if errors: logger.info(_('Bad launch configuration starting template {template_pk} as part of ' 'workflow {workflow_pk}. Errors:\n{error_text}').format( template_pk=ujt_obj.pk, workflow_pk=self.pk, error_text=errors)) data.update(accepted_fields) # missing fields are handled in the scheduler # build ancestor artifacts, save them to node model for later aa_dict = {} for parent_node in self.get_parent_nodes(): aa_dict.update(parent_node.ancestor_artifacts) if parent_node.job and hasattr(parent_node.job, 'artifacts'): aa_dict.update(parent_node.job.artifacts) if aa_dict: self.ancestor_artifacts = aa_dict self.save(update_fields=['ancestor_artifacts']) # process password list password_dict = {} if '_ansible_no_log' in aa_dict: for key in aa_dict: if key != '_ansible_no_log': password_dict[key] = REPLACE_STR if self.workflow_job.survey_passwords: password_dict.update(self.workflow_job.survey_passwords) if self.survey_passwords: password_dict.update(self.survey_passwords) if password_dict: data['survey_passwords'] = password_dict # process extra_vars extra_vars = data.get('extra_vars', {}) if aa_dict: functional_aa_dict = copy(aa_dict) functional_aa_dict.pop('_ansible_no_log', None) extra_vars.update(functional_aa_dict) # Workflow Job extra_vars higher precedence than ancestor artifacts if self.workflow_job and self.workflow_job.extra_vars: extra_vars.update(self.workflow_job.extra_vars_dict) if extra_vars: data['extra_vars'] = extra_vars # ensure that unified jobs created by WorkflowJobs are marked data['_eager_fields'] = {'launch_type': 'workflow'} return data
class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin): """A model representing a Queue/Group of AWX Instances.""" objects = InstanceGroupManager() name = models.CharField(max_length=250, unique=True) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) instances = models.ManyToManyField( 'Instance', related_name='rampart_groups', editable=False, help_text=_('Instances that are members of this InstanceGroup'), ) controller = models.ForeignKey( 'InstanceGroup', related_name='controlled_groups', help_text=_('Instance Group to remotely control this group.'), editable=False, default=None, null=True, on_delete=models.CASCADE) policy_instance_percentage = models.IntegerField( default=0, help_text=_( "Percentage of Instances to automatically assign to this group")) policy_instance_minimum = models.IntegerField( default=0, help_text= _("Static minimum number of Instances to automatically assign to this group" )) policy_instance_list = JSONField( default=[], blank=True, help_text= _("List of exact-match Instances that will always be automatically assigned to this group" )) POLICY_FIELDS = frozenset( ('policy_instance_list', 'policy_instance_minimum', 'policy_instance_percentage', 'controller')) def get_absolute_url(self, request=None): return reverse('api:instance_group_detail', kwargs={'pk': self.pk}, request=request) @property def capacity(self): return sum([inst.capacity for inst in self.instances.all()]) @property def jobs_running(self): return UnifiedJob.objects.filter(status__in=('running', 'waiting'), instance_group=self).count() @property def jobs_total(self): return UnifiedJob.objects.filter(instance_group=self).count() @property def is_controller(self): return self.controlled_groups.exists() @property def is_isolated(self): return bool(self.controller) ''' RelatedJobsMixin ''' def _get_related_jobs(self): return UnifiedJob.objects.filter(instance_group=self) class Meta: app_label = 'main' def fit_task_to_most_remaining_capacity_instance(self, task): instance_most_capacity = None for i in self.instances.filter(capacity__gt=0, enabled=True).order_by('hostname'): if i.remaining_capacity >= task.task_impact and \ (instance_most_capacity is None or i.remaining_capacity > instance_most_capacity.remaining_capacity): instance_most_capacity = i return instance_most_capacity def find_largest_idle_instance(self): largest_instance = None for i in self.instances.filter(capacity__gt=0, enabled=True).order_by('hostname'): if i.jobs_running == 0: if largest_instance is None: largest_instance = i elif i.capacity > largest_instance.capacity: largest_instance = i return largest_instance def choose_online_controller_node(self): return random.choice( list( self.controller.instances.filter( capacity__gt=0, enabled=True).values_list('hostname', flat=True)))
class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin): """ A project represents a playbook git repo that can access a set of inventories """ SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')] FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials'] FIELDS_TO_DISCARD_AT_COPY = ['local_path'] FIELDS_TRIGGER_UPDATE = frozenset(['scm_url', 'scm_branch', 'scm_type', 'scm_refspec']) class Meta: app_label = 'main' ordering = ('id',) default_environment = models.ForeignKey( 'ExecutionEnvironment', null=True, blank=True, default=None, on_delete=polymorphic.SET_NULL, related_name='+', help_text=_('The default execution environment for jobs run using this project.'), ) scm_update_on_launch = models.BooleanField( default=False, help_text=_('Update the project when a job is launched that uses the project.'), ) scm_update_cache_timeout = models.PositiveIntegerField( default=0, blank=True, help_text=_('The number of seconds after the last project update ran that a new ' 'project update will be launched as a job dependency.'), ) allow_override = models.BooleanField( default=False, help_text=_('Allow changing the SCM branch or revision in a job template ' 'that uses this project.'), ) scm_revision = models.CharField( max_length=1024, blank=True, default='', editable=False, verbose_name=_('SCM Revision'), help_text=_('The last revision fetched by a project update'), ) playbook_files = JSONField( blank=True, default=[], editable=False, verbose_name=_('Playbook Files'), help_text=_('List of playbooks found in the project'), ) inventory_files = JSONField( blank=True, default=[], editable=False, verbose_name=_('Inventory Files'), help_text=_('Suggested list of content that could be Ansible inventory in the project'), ) admin_role = ImplicitRoleField( parent_role=[ 'organization.project_admin_role', 'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ] ) use_role = ImplicitRoleField( parent_role='admin_role', ) update_role = ImplicitRoleField( parent_role='admin_role', ) read_role = ImplicitRoleField( parent_role=[ 'organization.auditor_role', 'singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR, 'use_role', 'update_role', ] ) @classmethod def _get_unified_job_class(cls): return ProjectUpdate @classmethod def _get_unified_job_field_names(cls): return set(f.name for f in ProjectOptions._meta.fields) | set(['name', 'description', 'organization']) def clean_organization(self): if self.pk: old_org_id = getattr(self, '_prior_values_store', {}).get('organization_id', None) if self.organization_id != old_org_id and self.jobtemplates.exists(): raise ValidationError({'organization': _('Organization cannot be changed when in use by job templates.')}) return self.organization def save(self, *args, **kwargs): new_instance = not bool(self.pk) pre_save_vals = getattr(self, '_prior_values_store', {}) # If update_fields has been specified, add our field names to it, # if it hasn't been specified, then we're just doing a normal save. update_fields = kwargs.get('update_fields', []) skip_update = bool(kwargs.pop('skip_update', False)) # Create auto-generated local path if project uses SCM. if self.pk and self.scm_type and not self.local_path.startswith('_'): slug_name = slugify(str(self.name)).replace(u'-', u'_') self.local_path = u'_%d__%s' % (int(self.pk), slug_name) if 'local_path' not in update_fields: update_fields.append('local_path') # Do the actual save. super(Project, self).save(*args, **kwargs) if new_instance: update_fields = [] # Generate local_path for SCM after initial save (so we have a PK). if self.scm_type and not self.local_path.startswith('_'): update_fields.append('local_path') if update_fields: from awx.main.signals import disable_activity_stream with disable_activity_stream(): self.save(update_fields=update_fields) # If we just created a new project with SCM, start the initial update. # also update if certain fields have changed relevant_change = any(pre_save_vals.get(fd_name, None) != self._prior_values_store.get(fd_name, None) for fd_name in self.FIELDS_TRIGGER_UPDATE) if (relevant_change or new_instance) and (not skip_update) and self.scm_type: self.update() def _get_current_status(self): if self.scm_type: if self.current_job and self.current_job.status: return self.current_job.status elif not self.last_job: return 'never updated' # inherit the child job status on failure elif self.last_job_failed: return self.last_job.status # Return the successful status else: return self.last_job.status elif not self.get_project_path(): return 'missing' else: return 'ok' def _get_last_job_run(self): if self.scm_type and self.last_job: return self.last_job.finished else: project_path = self.get_project_path() if project_path: try: mtime = os.path.getmtime(smart_str(project_path)) dt = datetime.datetime.fromtimestamp(mtime) return make_aware(dt, get_default_timezone()) except os.error: pass def _can_update(self): return bool(self.scm_type) def create_project_update(self, **kwargs): return self.create_unified_job(**kwargs) @property def cache_timeout_blocked(self): if not self.last_job_run: return False if (self.last_job_run + datetime.timedelta(seconds=self.scm_update_cache_timeout)) > now(): return True return False @property def needs_update_on_launch(self): if self.scm_type and self.scm_update_on_launch: if not self.last_job_run: return True if (self.last_job_run + datetime.timedelta(seconds=self.scm_update_cache_timeout)) <= now(): return True return False @property def cache_id(self): return str(self.last_job_id) @property def notification_templates(self): base_notification_templates = NotificationTemplate.objects error_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_errors=self)) started_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_started=self)) success_notification_templates = list(base_notification_templates.filter(unifiedjobtemplate_notification_templates_for_success=self)) # Get Organization NotificationTemplates if self.organization is not None: error_notification_templates = set( error_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_errors=self.organization)) ) started_notification_templates = set( started_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_started=self.organization)) ) success_notification_templates = set( success_notification_templates + list(base_notification_templates.filter(organization_notification_templates_for_success=self.organization)) ) return dict(error=list(error_notification_templates), started=list(started_notification_templates), success=list(success_notification_templates)) def get_absolute_url(self, request=None): return reverse('api:project_detail', kwargs={'pk': self.pk}, request=request) ''' RelatedJobsMixin ''' def _get_related_jobs(self): return UnifiedJob.objects.non_polymorphic().filter(models.Q(job__project=self) | models.Q(projectupdate__project=self)) def delete(self, *args, **kwargs): paths_to_delete = (self.get_project_path(check_if_exists=False), self.get_cache_path()) r = super(Project, self).delete(*args, **kwargs) for path_to_delete in paths_to_delete: if self.scm_type and path_to_delete: # non-manual, concrete path from awx.main.tasks import delete_project_files delete_project_files.delay(path_to_delete) return r
class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin, CustomVirtualEnvMixin): ''' A job applies a project (with playbook) to an inventory source with a given credential. It represents a single invocation of ansible-playbook with the given parameters. ''' class Meta: app_label = 'main' ordering = ('id', ) job_template = models.ForeignKey( 'JobTemplate', related_name='jobs', blank=True, null=True, default=None, on_delete=models.SET_NULL, ) hosts = models.ManyToManyField( 'Host', related_name='jobs', editable=False, through='JobHostSummary', ) artifacts = JSONField( blank=True, default={}, editable=False, ) scm_revision = models.CharField( max_length=1024, blank=True, default='', editable=False, verbose_name=_('SCM Revision'), help_text=_( 'The SCM Revision from the Project used for this job, if available' ), ) project_update = models.ForeignKey( 'ProjectUpdate', blank=True, null=True, default=None, on_delete=models.SET_NULL, help_text= _('The SCM Refresh task used to make sure the playbooks were available for the job run' ), ) job_slice_number = models.PositiveIntegerField( blank=True, default=0, help_text=_( "If part of a sliced job, the ID of the inventory slice operated on. " "If not part of sliced job, parameter is not used."), ) job_slice_count = models.PositiveIntegerField( blank=True, default=1, help_text=_( "If ran as part of sliced jobs, the total number of slices. " "If 1, job is not part of a sliced job."), ) def _get_parent_field_name(self): return 'job_template' @classmethod def _get_task_class(cls): from awx.main.tasks import RunJob return RunJob @classmethod def supports_isolation(cls): return True def _global_timeout_setting(self): return 'DEFAULT_JOB_TIMEOUT' @classmethod def _get_unified_job_template_class(cls): return JobTemplate def get_absolute_url(self, request=None): return reverse('api:job_detail', kwargs={'pk': self.pk}, request=request) def get_ui_url(self): return urljoin(settings.TOWER_URL_BASE, "/#/jobs/playbook/{}".format(self.pk)) @property def ansible_virtualenv_path(self): # the order here enforces precedence (it matters) for virtualenv in (self.job_template.custom_virtualenv if self.job_template else None, self.project.custom_virtualenv, self.project.organization.custom_virtualenv if self.project.organization else None): if virtualenv: return virtualenv return settings.ANSIBLE_VENV_PATH @property def event_class(self): return JobEvent def copy_unified_job(self, **new_prompts): # Needed for job slice relaunch consistency, do no re-spawn workflow job # target same slice as original job new_prompts['_prevent_slicing'] = True new_prompts.setdefault('_eager_fields', {}) new_prompts['_eager_fields'][ 'job_slice_number'] = self.job_slice_number new_prompts['_eager_fields']['job_slice_count'] = self.job_slice_count return super(Job, self).copy_unified_job(**new_prompts) @property def ask_diff_mode_on_launch(self): if self.job_template is not None: return self.job_template.ask_diff_mode_on_launch return False @property def ask_variables_on_launch(self): if self.job_template is not None: return self.job_template.ask_variables_on_launch return False @property def ask_limit_on_launch(self): if self.job_template is not None: return self.job_template.ask_limit_on_launch return False @property def ask_tags_on_launch(self): if self.job_template is not None: return self.job_template.ask_tags_on_launch return False @property def ask_skip_tags_on_launch(self): if self.job_template is not None: return self.job_template.ask_skip_tags_on_launch return False @property def ask_job_type_on_launch(self): if self.job_template is not None: return self.job_template.ask_job_type_on_launch return False @property def ask_verbosity_on_launch(self): if self.job_template is not None: return self.job_template.ask_verbosity_on_launch return False @property def ask_inventory_on_launch(self): if self.job_template is not None: return self.job_template.ask_inventory_on_launch return False @property def ask_credential_on_launch(self): if self.job_template is not None: return self.job_template.ask_credential_on_launch return False def get_passwords_needed_to_start(self): return self.passwords_needed_to_start def _get_hosts(self, **kwargs): Host = JobHostSummary._meta.get_field('host').related_model kwargs['job_host_summaries__job__pk'] = self.pk return Host.objects.filter(**kwargs) def retry_qs(self, status): ''' Returns Host queryset that will be used to produce the `limit` field in a retry on a subset of hosts ''' kwargs = {} if status == 'all': pass elif status == 'failed': # Special case for parity with Ansible .retry files kwargs['job_host_summaries__failed'] = True elif status in ['ok', 'changed', 'unreachable']: if status == 'unreachable': status_field = 'dark' else: status_field = status kwargs['job_host_summaries__{}__gt'.format(status_field)] = 0 else: raise ParseError( _('{status_value} is not a valid status option.').format( status_value=status)) return self._get_hosts(**kwargs) @property def task_impact(self): # NOTE: We sorta have to assume the host count matches and that forks default to 5 from awx.main.models.inventory import Host if self.launch_type == 'callback': count_hosts = 2 else: count_hosts = Host.objects.filter( inventory__jobs__pk=self.pk).count() if self.job_slice_count > 1: # Integer division intentional count_hosts = (count_hosts + self.job_slice_count - self.job_slice_number) // self.job_slice_count return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1 @property def successful_hosts(self): return self._get_hosts(job_host_summaries__ok__gt=0) @property def failed_hosts(self): return self._get_hosts(job_host_summaries__failures__gt=0) @property def changed_hosts(self): return self._get_hosts(job_host_summaries__changed__gt=0) @property def dark_hosts(self): return self._get_hosts(job_host_summaries__dark__gt=0) @property def unreachable_hosts(self): return self.dark_hosts @property def skipped_hosts(self): return self._get_hosts(job_host_summaries__skipped__gt=0) @property def processed_hosts(self): return self._get_hosts(job_host_summaries__processed__gt=0) def notification_data(self, block=5): data = super(Job, self).notification_data() all_hosts = {} # NOTE: Probably related to job event slowness, remove at some point -matburt if block: summaries = self.job_host_summaries.all() while block > 0 and not len(summaries): time.sleep(1) block -= 1 else: summaries = self.job_host_summaries.all() for h in self.job_host_summaries.all(): all_hosts[h.host_name] = dict(failed=h.failed, changed=h.changed, dark=h.dark, failures=h.failures, ok=h.ok, processed=h.processed, skipped=h.skipped) data.update( dict(inventory=self.inventory.name if self.inventory else None, project=self.project.name if self.project else None, playbook=self.playbook, credential=getattr(self.get_deprecated_credential('ssh'), 'name', None), limit=self.limit, extra_vars=self.display_extra_vars(), hosts=all_hosts)) return data def _resources_sufficient_for_launch(self): return not (self.inventory_id is None or self.project_id is None) def display_artifacts(self): ''' Hides artifacts if they are marked as no_log type artifacts. ''' artifacts = self.artifacts if artifacts.get('_ansible_no_log', False): return "$hidden due to Ansible no_log flag$" return artifacts @property def preferred_instance_groups(self): if self.project is not None and self.project.organization is not None: organization_groups = [ x for x in self.project.organization.instance_groups.all() ] else: organization_groups = [] if self.inventory is not None: inventory_groups = [ x for x in self.inventory.instance_groups.all() ] else: inventory_groups = [] if self.job_template is not None: template_groups = [ x for x in self.job_template.instance_groups.all() ] else: template_groups = [] selected_groups = template_groups + inventory_groups + organization_groups if not selected_groups: return self.global_instance_groups return selected_groups def awx_meta_vars(self): r = super(Job, self).awx_meta_vars() if self.project: for name in ('awx', 'tower'): r['{}_project_revision'.format( name)] = self.project.scm_revision if self.job_template: for name in ('awx', 'tower'): r['{}_job_template_id'.format(name)] = self.job_template.pk r['{}_job_template_name'.format(name)] = self.job_template.name return r ''' JobNotificationMixin ''' def get_notification_templates(self): if not self.job_template: return NotificationTemplate.objects.none() return self.job_template.notification_templates def get_notification_friendly_name(self): return "Job" def _get_inventory_hosts(self, only=[ 'name', 'ansible_facts', 'ansible_facts_modified', 'modified', 'inventory_id' ]): if not self.inventory: return [] return self.inventory.hosts.only(*only) def start_job_fact_cache(self, destination, modification_times, timeout=None): destination = os.path.join(destination, 'facts') os.makedirs(destination, mode=0o700) hosts = self._get_inventory_hosts() if timeout is None: timeout = settings.ANSIBLE_FACT_CACHE_TIMEOUT if timeout > 0: # exclude hosts with fact data older than `settings.ANSIBLE_FACT_CACHE_TIMEOUT seconds` timeout = now() - datetime.timedelta(seconds=timeout) hosts = hosts.filter(ansible_facts_modified__gte=timeout) for host in hosts: filepath = os.sep.join(map(str, [destination, host.name])) if not os.path.realpath(filepath).startswith(destination): system_tracking_logger.error( 'facts for host {} could not be cached'.format( smart_str(host.name))) continue try: with codecs.open(filepath, 'w', encoding='utf-8') as f: os.chmod(f.name, 0o600) json.dump(host.ansible_facts, f) except IOError: system_tracking_logger.error( 'facts for host {} could not be cached'.format( smart_str(host.name))) continue # make note of the time we wrote the file so we can check if it changed later modification_times[filepath] = os.path.getmtime(filepath) def finish_job_fact_cache(self, destination, modification_times): destination = os.path.join(destination, 'facts') for host in self._get_inventory_hosts(): filepath = os.sep.join(map(str, [destination, host.name])) if not os.path.realpath(filepath).startswith(destination): system_tracking_logger.error( 'facts for host {} could not be cached'.format( smart_str(host.name))) continue if os.path.exists(filepath): # If the file changed since we wrote it pre-playbook run... modified = os.path.getmtime(filepath) if modified > modification_times.get(filepath, 0): with codecs.open(filepath, 'r', encoding='utf-8') as f: try: ansible_facts = json.load(f) except ValueError: continue host.ansible_facts = ansible_facts host.ansible_facts_modified = now() if 'insights' in ansible_facts and 'system_id' in ansible_facts[ 'insights']: host.insights_system_id = ansible_facts[ 'insights']['system_id'] host.save() system_tracking_logger.info( 'New fact for inventory {} host {}'.format( smart_str(host.inventory.name), smart_str(host.name)), extra=dict(inventory_id=host.inventory.id, host_name=host.name, ansible_facts=host.ansible_facts, ansible_facts_modified=host. ansible_facts_modified.isoformat(), job_id=self.id)) else: # if the file goes missing, ansible removed it (likely via clear_facts) host.ansible_facts = {} host.ansible_facts_modified = now() system_tracking_logger.info( 'Facts cleared for inventory {} host {}'.format( smart_str(host.inventory.name), smart_str(host.name))) host.save()
class SurveyJobTemplateMixin(models.Model): class Meta: abstract = True survey_enabled = models.BooleanField(default=False, ) survey_spec = prevent_search(JSONField( blank=True, default=dict, )) ask_variables_on_launch = AskForField(blank=True, default=False, allows_field='extra_vars') def survey_password_variables(self): vars = [] if self.survey_enabled and 'spec' in self.survey_spec: # Get variables that are type password for survey_element in self.survey_spec['spec']: if survey_element['type'] == 'password': vars.append(survey_element['variable']) return vars @property def variables_needed_to_start(self): vars = [] if self.survey_enabled and 'spec' in self.survey_spec: for survey_element in self.survey_spec['spec']: if survey_element['required']: vars.append(survey_element['variable']) return vars def _update_unified_job_kwargs(self, create_kwargs, kwargs): ''' Combine extra_vars with variable precedence order: JT extra_vars -> JT survey defaults -> runtime extra_vars :param create_kwargs: key-worded arguments to be updated and later used for creating unified job. :type create_kwargs: dict :param kwargs: request parameters used to override unified job template fields with runtime values. :type kwargs: dict :return: modified create_kwargs. :rtype: dict ''' # Job Template extra_vars extra_vars = self.extra_vars_dict survey_defaults = {} # transform to dict if 'extra_vars' in kwargs: runtime_extra_vars = kwargs['extra_vars'] runtime_extra_vars = parse_yaml_or_json(runtime_extra_vars) else: runtime_extra_vars = {} # Overwrite job template extra vars with survey default vars if self.survey_enabled and 'spec' in self.survey_spec: for survey_element in self.survey_spec.get("spec", []): default = survey_element.get('default') variable_key = survey_element.get('variable') if survey_element.get('type') == 'password': if variable_key in runtime_extra_vars: kw_value = runtime_extra_vars[variable_key] if kw_value == '$encrypted$': runtime_extra_vars.pop(variable_key) if default is not None: decrypted_default = default if (survey_element['type'] == "password" and isinstance(decrypted_default, str) and decrypted_default.startswith('$encrypted$')): decrypted_default = decrypt_value( get_encryption_key('value', pk=None), decrypted_default) errors = self._survey_element_validation( survey_element, {variable_key: decrypted_default}) if not errors: survey_defaults[variable_key] = default extra_vars.update(survey_defaults) # Overwrite job template extra vars with explicit job extra vars # and add on job extra vars extra_vars.update(runtime_extra_vars) create_kwargs['extra_vars'] = json.dumps(extra_vars) return create_kwargs def _survey_element_validation(self, survey_element, data, validate_required=True): # Don't apply validation to the `$encrypted$` placeholder; the decrypted # default (if any) will be validated against instead errors = [] if (survey_element['type'] == "password"): password_value = data.get(survey_element['variable']) if (isinstance(password_value, str) and password_value == '$encrypted$'): if survey_element.get( 'default') is None and survey_element['required']: if validate_required: errors.append("'%s' value missing" % survey_element['variable']) return errors if survey_element['variable'] not in data and survey_element[ 'required']: if validate_required: errors.append("'%s' value missing" % survey_element['variable']) elif survey_element['type'] in ["textarea", "text", "password"]: if survey_element['variable'] in data: if not isinstance(data[survey_element['variable']], str): errors.append( "Value %s for '%s' expected to be a string." % (data[survey_element['variable']], survey_element['variable'])) return errors if 'min' in survey_element and survey_element['min'] not in [ "", None ] and len(data[survey_element['variable']]) < int( survey_element['min']): errors.append( "'%s' value %s is too small (length is %s must be at least %s)." % (survey_element['variable'], data[survey_element['variable']], len(data[survey_element['variable']]), survey_element['min'])) if 'max' in survey_element and survey_element['max'] not in [ "", None ] and len(data[survey_element['variable']]) > int( survey_element['max']): errors.append( "'%s' value %s is too large (must be no more than %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) elif survey_element['type'] == 'integer': if survey_element['variable'] in data: if type(data[survey_element['variable']]) != int: errors.append( "Value %s for '%s' expected to be an integer." % (data[survey_element['variable']], survey_element['variable'])) return errors if 'min' in survey_element and survey_element['min'] not in ["", None] and survey_element['variable'] in data and \ data[survey_element['variable']] < int(survey_element['min']): errors.append( "'%s' value %s is too small (must be at least %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['min'])) if 'max' in survey_element and survey_element['max'] not in ["", None] and survey_element['variable'] in data and \ data[survey_element['variable']] > int(survey_element['max']): errors.append( "'%s' value %s is too large (must be no more than %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) elif survey_element['type'] == 'float': if survey_element['variable'] in data: if type(data[survey_element['variable']]) not in (float, int): errors.append( "Value %s for '%s' expected to be a numeric type." % (data[survey_element['variable']], survey_element['variable'])) return errors if 'min' in survey_element and survey_element['min'] not in [ "", None ] and data[survey_element['variable']] < float( survey_element['min']): errors.append( "'%s' value %s is too small (must be at least %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['min'])) if 'max' in survey_element and survey_element['max'] not in [ "", None ] and data[survey_element['variable']] > float( survey_element['max']): errors.append( "'%s' value %s is too large (must be no more than %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) elif survey_element['type'] == 'multiselect': if survey_element['variable'] in data: if type(data[survey_element['variable']]) != list: errors.append("'%s' value is expected to be a list." % survey_element['variable']) else: choice_list = copy(survey_element['choices']) if isinstance(choice_list, str): choice_list = [ choice for choice in choice_list.splitlines() if choice.strip() != '' ] for val in data[survey_element['variable']]: if val not in choice_list: errors.append( "Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'], choice_list)) elif survey_element['type'] == 'multiplechoice': choice_list = copy(survey_element['choices']) if isinstance(choice_list, str): choice_list = [ choice for choice in choice_list.splitlines() if choice.strip() != '' ] if survey_element['variable'] in data: if data[survey_element['variable']] not in choice_list: errors.append( "Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']], survey_element['variable'], choice_list)) return errors def _accept_or_ignore_variables(self, data, errors=None, _exclude_errors=(), extra_passwords=None): survey_is_enabled = (self.survey_enabled and self.survey_spec) extra_vars = data.copy() if errors is None: errors = {} rejected = {} accepted = {} if survey_is_enabled: # Check for data violation of survey rules survey_errors = [] for survey_element in self.survey_spec.get("spec", []): key = survey_element.get('variable', None) value = data.get(key, None) validate_required = 'required' not in _exclude_errors if extra_passwords and key in extra_passwords and is_encrypted( value): element_errors = self._survey_element_validation( survey_element, { key: decrypt_value(get_encryption_key('value', pk=None), value) }, validate_required=validate_required) else: element_errors = self._survey_element_validation( survey_element, data, validate_required=validate_required) if element_errors: survey_errors += element_errors if key is not None and key in extra_vars: rejected[key] = extra_vars.pop(key) elif key in extra_vars: accepted[key] = extra_vars.pop(key) if survey_errors: errors['variables_needed_to_start'] = survey_errors if self.ask_variables_on_launch: # We can accept all variables accepted.update(extra_vars) extra_vars = {} if extra_vars: # Prune the prompted variables for those identical to template tmp_extra_vars = self.extra_vars_dict for key in (set(tmp_extra_vars.keys()) & set(extra_vars.keys())): if tmp_extra_vars[key] == extra_vars[key]: extra_vars.pop(key) if extra_vars: # Leftover extra_vars, keys provided that are not allowed rejected.update(extra_vars) # ignored variables does not block manual launch if 'prompts' not in _exclude_errors: errors['extra_vars'] = [ _('Variables {list_of_keys} are not allowed on launch. Check the Prompt on Launch setting ' + 'on the {model_name} to include Extra Variables.'). format(list_of_keys=', '.join( [str(key) for key in extra_vars.keys()]), model_name=self._meta.verbose_name.title()) ] return (accepted, rejected, errors) @staticmethod def pivot_spec(spec): ''' Utility method that will return a dictionary keyed off variable names ''' pivoted = {} for element_data in spec.get('spec', []): if 'variable' in element_data: pivoted[element_data['variable']] = element_data return pivoted def survey_variable_validation(self, data): errors = [] if not self.survey_enabled: return errors if 'name' not in self.survey_spec: errors.append("'name' missing from survey spec.") if 'description' not in self.survey_spec: errors.append("'description' missing from survey spec.") for survey_element in self.survey_spec.get("spec", []): errors += self._survey_element_validation(survey_element, data) return errors def display_survey_spec(self): ''' Hide encrypted default passwords in survey specs ''' survey_spec = deepcopy(self.survey_spec) if self.survey_spec else {} for field in survey_spec.get('spec', []): if field.get('type') == 'password': if 'default' in field and field['default']: field['default'] = '$encrypted$' return survey_spec
class NotificationTemplate(CommonModelNameNotUnique): NOTIFICATION_TYPES = [('email', _('Email'), CustomEmailBackend), ('slack', _('Slack'), SlackBackend), ('twilio', _('Twilio'), TwilioBackend), ('pagerduty', _('Pagerduty'), PagerDutyBackend), ('hipchat', _('HipChat'), HipChatBackend), ('webhook', _('Webhook'), WebhookBackend), ('mattermost', _('Mattermost'), MattermostBackend), ('irc', _('IRC'), IrcBackend)] NOTIFICATION_TYPE_CHOICES = [(x[0], x[1]) for x in NOTIFICATION_TYPES] CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES]) class Meta: app_label = 'main' unique_together = ('organization', 'name') organization = models.ForeignKey( 'Organization', blank=False, null=True, on_delete=models.CASCADE, related_name='notification_templates', ) notification_type = models.CharField( max_length=32, choices=NOTIFICATION_TYPE_CHOICES, ) notification_configuration = JSONField(blank=False) def get_absolute_url(self, request=None): return reverse('api:notification_template_detail', kwargs={'pk': self.pk}, request=request) @property def notification_class(self): return self.CLASS_FOR_NOTIFICATION_TYPE[self.notification_type] def save(self, *args, **kwargs): new_instance = not bool(self.pk) update_fields = kwargs.get('update_fields', []) for field in filter( lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters): if self.notification_configuration[field].startswith( "$encrypted$"): continue if new_instance: value = self.notification_configuration[field] setattr(self, '_saved_{}_{}'.format("config", field), value) self.notification_configuration[field] = '' else: encrypted = encrypt_field(self, 'notification_configuration', subfield=field, skip_utf8=True) self.notification_configuration[field] = encrypted if 'notification_configuration' not in update_fields: update_fields.append('notification_configuration') super(NotificationTemplate, self).save(*args, **kwargs) if new_instance: update_fields = [] for field in filter( lambda x: self.notification_class.init_parameters[x][ 'type'] == "password", self.notification_class.init_parameters): saved_value = getattr(self, '_saved_{}_{}'.format("config", field), '') self.notification_configuration[field] = saved_value if 'notification_configuration' not in update_fields: update_fields.append('notification_configuration') self.save(update_fields=update_fields) @property def recipients(self): return self.notification_configuration[ self.notification_class.recipient_parameter] def generate_notification(self, subject, message): notification = Notification(notification_template=self, notification_type=self.notification_type, recipients=smart_str(self.recipients), subject=subject, body=message) notification.save() return notification def send(self, subject, body): for field in filter( lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters): self.notification_configuration[field] = decrypt_field( self, 'notification_configuration', subfield=field) recipients = self.notification_configuration.pop( self.notification_class.recipient_parameter) if not isinstance(recipients, list): recipients = [recipients] sender = self.notification_configuration.pop( self.notification_class.sender_parameter, None) backend_obj = self.notification_class( **self.notification_configuration) notification_obj = EmailMessage(subject, backend_obj.format_body(body), sender, recipients) with set_environ(**settings.AWX_TASK_ENV): return backend_obj.send_messages([notification_obj]) def display_notification_configuration(self): field_val = self.notification_configuration.copy() for field in self.notification_class.init_parameters: if field in field_val and force_text( field_val[field]).startswith('$encrypted$'): field_val[field] = '$encrypted$' return field_val
class ActivityStream(models.Model): """ Model used to describe activity stream (audit) events """ class Meta: app_label = 'main' ordering = ('pk', ) OPERATION_CHOICES = [ ('create', _('Entity Created')), ('update', _("Entity Updated")), ('delete', _("Entity Deleted")), ('associate', _("Entity Associated with another Entity")), ('disassociate', _("Entity was Disassociated with another Entity")), ] actor = models.ForeignKey('auth.User', null=True, on_delete=models.SET_NULL, related_name='activity_stream') operation = models.CharField(max_length=13, choices=OPERATION_CHOICES) timestamp = models.DateTimeField(auto_now_add=True) changes = accepts_json(models.TextField(blank=True)) deleted_actor = JSONField(null=True) action_node = models.CharField( blank=True, default='', editable=False, max_length=512, help_text=_("The cluster node the activity took place on."), ) object_relationship_type = models.TextField(blank=True) object1 = models.TextField() object2 = models.TextField() user = models.ManyToManyField("auth.User", blank=True) organization = models.ManyToManyField("Organization", blank=True) inventory = models.ManyToManyField("Inventory", blank=True) host = models.ManyToManyField("Host", blank=True) group = models.ManyToManyField("Group", blank=True) inventory_source = models.ManyToManyField("InventorySource", blank=True) inventory_update = models.ManyToManyField("InventoryUpdate", blank=True) credential = models.ManyToManyField("Credential", blank=True) credential_type = models.ManyToManyField("CredentialType", blank=True) team = models.ManyToManyField("Team", blank=True) project = models.ManyToManyField("Project", blank=True) project_update = models.ManyToManyField("ProjectUpdate", blank=True) execution_environment = models.ManyToManyField("ExecutionEnvironment", blank=True) job_template = models.ManyToManyField("JobTemplate", blank=True) job = models.ManyToManyField("Job", blank=True) workflow_job_template_node = models.ManyToManyField( "WorkflowJobTemplateNode", blank=True) workflow_job_node = models.ManyToManyField("WorkflowJobNode", blank=True) workflow_job_template = models.ManyToManyField("WorkflowJobTemplate", blank=True) workflow_job = models.ManyToManyField("WorkflowJob", blank=True) workflow_approval_template = models.ManyToManyField( "WorkflowApprovalTemplate", blank=True) workflow_approval = models.ManyToManyField("WorkflowApproval", blank=True) unified_job_template = models.ManyToManyField( "UnifiedJobTemplate", blank=True, related_name='activity_stream_as_unified_job_template+') unified_job = models.ManyToManyField( "UnifiedJob", blank=True, related_name='activity_stream_as_unified_job+') ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True) schedule = models.ManyToManyField("Schedule", blank=True) execution_environment = models.ManyToManyField("ExecutionEnvironment", blank=True) notification_template = models.ManyToManyField("NotificationTemplate", blank=True) notification = models.ManyToManyField("Notification", blank=True) label = models.ManyToManyField("Label", blank=True) role = models.ManyToManyField("Role", blank=True) instance = models.ManyToManyField("Instance", blank=True) instance_group = models.ManyToManyField("InstanceGroup", blank=True) o_auth2_application = models.ManyToManyField("OAuth2Application", blank=True) o_auth2_access_token = models.ManyToManyField("OAuth2AccessToken", blank=True) setting = JSONField(blank=True) def __str__(self): operation = self.operation if 'operation' in self.__dict__ else '_delayed_' if 'timestamp' in self.__dict__: if self.timestamp: timestamp = self.timestamp.isoformat() else: timestamp = self.timestamp else: timestamp = '_delayed_' return u'%s-%s-pk=%s' % (operation, timestamp, self.pk) def get_absolute_url(self, request=None): return reverse('api:activity_stream_detail', kwargs={'pk': self.pk}, request=request) def save(self, *args, **kwargs): # Store denormalized actor metadata so that we retain it for accounting # purposes when the User row is deleted. if self.actor: self.deleted_actor = { 'id': self.actor_id, 'username': smart_str(self.actor.username), 'first_name': smart_str(self.actor.first_name), 'last_name': smart_str(self.actor.last_name), } if 'update_fields' in kwargs and 'deleted_actor' not in kwargs[ 'update_fields']: kwargs['update_fields'].append('deleted_actor') hostname_char_limit = self._meta.get_field('action_node').max_length self.action_node = settings.CLUSTER_HOST_ID[:hostname_char_limit] super(ActivityStream, self).save(*args, **kwargs)
class SurveyJobTemplateMixin(models.Model): class Meta: abstract = True survey_enabled = models.BooleanField(default=False, ) survey_spec = prevent_search(JSONField( blank=True, default={}, )) ask_variables_on_launch = AskForField(blank=True, default=False, allows_field='extra_vars') def survey_password_variables(self): vars = [] if self.survey_enabled and 'spec' in self.survey_spec: # Get variables that are type password for survey_element in self.survey_spec['spec']: if survey_element['type'] == 'password': vars.append(survey_element['variable']) return vars @property def variables_needed_to_start(self): vars = [] if self.survey_enabled and 'spec' in self.survey_spec: for survey_element in self.survey_spec['spec']: if survey_element['required']: vars.append(survey_element['variable']) return vars def _update_unified_job_kwargs(self, create_kwargs, kwargs): ''' Combine extra_vars with variable precedence order: JT extra_vars -> JT survey defaults -> runtime extra_vars :param create_kwargs: key-worded arguments to be updated and later used for creating unified job. :type create_kwargs: dict :param kwargs: request parameters used to override unified job template fields with runtime values. :type kwargs: dict :return: modified create_kwargs. :rtype: dict ''' # Job Template extra_vars extra_vars = self.extra_vars_dict survey_defaults = {} # transform to dict if 'extra_vars' in kwargs: runtime_extra_vars = kwargs['extra_vars'] runtime_extra_vars = parse_yaml_or_json(runtime_extra_vars) else: runtime_extra_vars = {} # Overwrite with job template extra vars with survey default vars if self.survey_enabled and 'spec' in self.survey_spec: for survey_element in self.survey_spec.get("spec", []): default = survey_element.get('default') variable_key = survey_element.get('variable') if survey_element.get('type') == 'password': if variable_key in runtime_extra_vars and default: kw_value = runtime_extra_vars[variable_key] if kw_value.startswith( '$encrypted$') and kw_value != default: runtime_extra_vars[variable_key] = default if default is not None: data = {variable_key: default} errors = self._survey_element_validation( survey_element, data) if not errors: survey_defaults[variable_key] = default extra_vars.update(survey_defaults) # Overwrite job template extra vars with explicit job extra vars # and add on job extra vars extra_vars.update(runtime_extra_vars) create_kwargs['extra_vars'] = json.dumps(extra_vars) return create_kwargs def _survey_element_validation(self, survey_element, data): errors = [] if survey_element['variable'] not in data and survey_element[ 'required']: errors.append("'%s' value missing" % survey_element['variable']) elif survey_element['type'] in ["textarea", "text", "password"]: if survey_element['variable'] in data: if type(data[survey_element['variable']]) not in (str, unicode): errors.append( "Value %s for '%s' expected to be a string." % (data[survey_element['variable']], survey_element['variable'])) return errors if 'min' in survey_element and survey_element['min'] not in [ "", None ] and len(data[survey_element['variable']]) < int( survey_element['min']): errors.append( "'%s' value %s is too small (length is %s must be at least %s)." % (survey_element['variable'], data[survey_element['variable']], len(data[survey_element['variable']]), survey_element['min'])) if 'max' in survey_element and survey_element['max'] not in [ "", None ] and len(data[survey_element['variable']]) > int( survey_element['max']): errors.append( "'%s' value %s is too large (must be no more than %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) elif survey_element['type'] == 'integer': if survey_element['variable'] in data: if type(data[survey_element['variable']]) != int: errors.append( "Value %s for '%s' expected to be an integer." % (data[survey_element['variable']], survey_element['variable'])) return errors if 'min' in survey_element and survey_element['min'] not in ["", None] and survey_element['variable'] in data and \ data[survey_element['variable']] < int(survey_element['min']): errors.append( "'%s' value %s is too small (must be at least %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['min'])) if 'max' in survey_element and survey_element['max'] not in ["", None] and survey_element['variable'] in data and \ data[survey_element['variable']] > int(survey_element['max']): errors.append( "'%s' value %s is too large (must be no more than %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) elif survey_element['type'] == 'float': if survey_element['variable'] in data: if type(data[survey_element['variable']]) not in (float, int): errors.append( "Value %s for '%s' expected to be a numeric type." % (data[survey_element['variable']], survey_element['variable'])) return errors if 'min' in survey_element and survey_element['min'] not in [ "", None ] and data[survey_element['variable']] < float( survey_element['min']): errors.append( "'%s' value %s is too small (must be at least %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['min'])) if 'max' in survey_element and survey_element['max'] not in [ "", None ] and data[survey_element['variable']] > float( survey_element['max']): errors.append( "'%s' value %s is too large (must be no more than %s)." % (survey_element['variable'], data[survey_element['variable']], survey_element['max'])) elif survey_element['type'] == 'multiselect': if survey_element['variable'] in data: if type(data[survey_element['variable']]) != list: errors.append("'%s' value is expected to be a list." % survey_element['variable']) else: choice_list = copy(survey_element['choices']) if isinstance(choice_list, basestring): choice_list = choice_list.split('\n') for val in data[survey_element['variable']]: if val not in choice_list: errors.append( "Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'], choice_list)) elif survey_element['type'] == 'multiplechoice': choice_list = copy(survey_element['choices']) if isinstance(choice_list, basestring): choice_list = choice_list.split('\n') if survey_element['variable'] in data: if data[survey_element['variable']] not in choice_list: errors.append( "Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']], survey_element['variable'], choice_list)) return errors def _accept_or_ignore_variables(self, data, errors=None, _exclude_errors=()): survey_is_enabled = (self.survey_enabled and self.survey_spec) extra_vars = data.copy() if errors is None: errors = {} rejected = {} accepted = {} if survey_is_enabled: # Check for data violation of survey rules survey_errors = [] for survey_element in self.survey_spec.get("spec", []): element_errors = self._survey_element_validation( survey_element, data) key = survey_element.get('variable', None) if element_errors: survey_errors += element_errors if key is not None and key in extra_vars: rejected[key] = extra_vars.pop(key) elif key in extra_vars: accepted[key] = extra_vars.pop(key) if survey_errors: errors['variables_needed_to_start'] = survey_errors if self.ask_variables_on_launch: # We can accept all variables accepted.update(extra_vars) extra_vars = {} if extra_vars: # Leftover extra_vars, keys provided that are not allowed rejected.update(extra_vars) # ignored variables does not block manual launch if 'prompts' not in _exclude_errors: errors['extra_vars'] = [ _('Variables {list_of_keys} are not allowed on launch.'). format(list_of_keys=', '.join(extra_vars.keys())) ] return (accepted, rejected, errors)
class LaunchTimeConfig(BaseModel): ''' Common model for all objects that save details of a saved launch config WFJT / WJ nodes, schedules, and job launch configs (not all implemented yet) ''' class Meta: abstract = True # Prompting-related fields that have to be handled as special cases credentials = models.ManyToManyField('Credential', related_name='%(class)ss') inventory = models.ForeignKey( 'Inventory', related_name='%(class)ss', blank=True, null=True, default=None, on_delete=models.SET_NULL, ) extra_data = JSONField(blank=True, default={}) survey_passwords = prevent_search( JSONField( blank=True, default={}, editable=False, )) # All standard fields are stored in this dictionary field # This is a solution to the nullable CharField problem, specific to prompting char_prompts = JSONField(blank=True, default={}) def prompts_dict(self, display=False): data = {} for prompt_name in JobTemplate.get_ask_mapping().keys(): try: field = self._meta.get_field(prompt_name) except FieldDoesNotExist: field = None if isinstance(field, models.ManyToManyField): if not self.pk: continue # unsaved object can't have related many-to-many prompt_val = set(getattr(self, prompt_name).all()) if len(prompt_val) > 0: data[prompt_name] = prompt_val elif prompt_name == 'extra_vars': if self.extra_data: if display: data[prompt_name] = self.display_extra_data() else: data[prompt_name] = self.extra_data if self.survey_passwords and not display: data['survey_passwords'] = self.survey_passwords else: prompt_val = getattr(self, prompt_name) if prompt_val is not None: data[prompt_name] = prompt_val return data def display_extra_data(self): ''' Hides fields marked as passwords in survey. ''' if self.survey_passwords: extra_data = parse_yaml_or_json(self.extra_data).copy() for key, value in self.survey_passwords.items(): if key in extra_data: extra_data[key] = value return extra_data else: return self.extra_data @property def _credential(self): ''' Only used for workflow nodes to support backward compatibility. ''' try: return [ cred for cred in self.credentials.all() if cred.credential_type.kind == 'ssh' ][0] except IndexError: return None @property def credential(self): ''' Returns an integer so it can be used as IntegerField in serializer ''' cred = self._credential if cred is not None: return cred.pk else: return None
class NotificationTemplate(CommonModelNameNotUnique): NOTIFICATION_TYPES = [('email', _('Email'), CustomEmailBackend), ('slack', _('Slack'), SlackBackend), ('twilio', _('Twilio'), TwilioBackend), ('pagerduty', _('Pagerduty'), PagerDutyBackend), ('grafana', _('Grafana'), GrafanaBackend), ('hipchat', _('HipChat'), HipChatBackend), ('webhook', _('Webhook'), WebhookBackend), ('mattermost', _('Mattermost'), MattermostBackend), ('rocketchat', _('Rocket.Chat'), RocketChatBackend), ('irc', _('IRC'), IrcBackend)] NOTIFICATION_TYPE_CHOICES = sorted([(x[0], x[1]) for x in NOTIFICATION_TYPES]) CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES]) class Meta: app_label = 'main' unique_together = ('organization', 'name') ordering = ("name", ) organization = models.ForeignKey( 'Organization', blank=False, null=True, on_delete=models.CASCADE, related_name='notification_templates', ) notification_type = models.CharField( max_length=32, choices=NOTIFICATION_TYPE_CHOICES, ) notification_configuration = prevent_search(JSONField(blank=False)) def default_messages(): return { 'started': None, 'success': None, 'error': None, 'workflow_approval': None } messages = JSONField( null=True, blank=True, default=default_messages, help_text=_('Optional custom messages for notification template.')) def has_message(self, condition): potential_template = self.messages.get(condition, {}) if potential_template == {}: return False if potential_template.get('message', {}) == {}: return False return True def get_message(self, condition): return self.messages.get(condition, {}) def get_absolute_url(self, request=None): return reverse('api:notification_template_detail', kwargs={'pk': self.pk}, request=request) @property def notification_class(self): return self.CLASS_FOR_NOTIFICATION_TYPE[self.notification_type] def save(self, *args, **kwargs): new_instance = not bool(self.pk) update_fields = kwargs.get('update_fields', []) # preserve existing notification messages if not overwritten by new messages if not new_instance: old_nt = NotificationTemplate.objects.get(pk=self.id) old_messages = old_nt.messages new_messages = self.messages def merge_messages(local_old_messages, local_new_messages, local_event): if local_new_messages.get(local_event, {}) and local_old_messages.get( local_event, {}): local_old_event_msgs = local_old_messages[local_event] local_new_event_msgs = local_new_messages[local_event] for msg_type in ['message', 'body']: if msg_type not in local_new_event_msgs and local_old_event_msgs.get( msg_type, None): local_new_event_msgs[ msg_type] = local_old_event_msgs[msg_type] if old_messages is not None and new_messages is not None: for event in ('started', 'success', 'error', 'workflow_approval'): if not new_messages.get(event, {}) and old_messages.get( event, {}): new_messages[event] = old_messages[event] continue if event == 'workflow_approval' and old_messages.get( 'workflow_approval', None): new_messages.setdefault('workflow_approval', {}) for subevent in ('running', 'approved', 'timed_out', 'denied'): old_wfa_messages = old_messages[ 'workflow_approval'] new_wfa_messages = new_messages[ 'workflow_approval'] if not new_wfa_messages.get( subevent, {}) and old_wfa_messages.get( subevent, {}): new_wfa_messages[subevent] = old_wfa_messages[ subevent] continue if old_wfa_messages: merge_messages(old_wfa_messages, new_wfa_messages, subevent) else: merge_messages(old_messages, new_messages, event) new_messages.setdefault(event, None) for field in filter( lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters): if self.notification_configuration[field].startswith( "$encrypted$"): continue if new_instance: value = self.notification_configuration[field] setattr(self, '_saved_{}_{}'.format("config", field), value) self.notification_configuration[field] = '' else: encrypted = encrypt_field(self, 'notification_configuration', subfield=field) self.notification_configuration[field] = encrypted if 'notification_configuration' not in update_fields: update_fields.append('notification_configuration') super(NotificationTemplate, self).save(*args, **kwargs) if new_instance: update_fields = [] for field in filter( lambda x: self.notification_class.init_parameters[x][ 'type'] == "password", self.notification_class.init_parameters): saved_value = getattr(self, '_saved_{}_{}'.format("config", field), '') self.notification_configuration[field] = saved_value if 'notification_configuration' not in update_fields: update_fields.append('notification_configuration') self.save(update_fields=update_fields) @property def recipients(self): return self.notification_configuration[ self.notification_class.recipient_parameter] def generate_notification(self, msg, body): notification = Notification(notification_template=self, notification_type=self.notification_type, recipients=smart_str(self.recipients), subject=msg, body=body) notification.save() return notification def send(self, subject, body): for field in filter( lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters): if field in self.notification_configuration: self.notification_configuration[field] = decrypt_field( self, 'notification_configuration', subfield=field) recipients = self.notification_configuration.pop( self.notification_class.recipient_parameter) if not isinstance(recipients, list): recipients = [recipients] sender = self.notification_configuration.pop( self.notification_class.sender_parameter, None) notification_configuration = deepcopy(self.notification_configuration) for field, params in self.notification_class.init_parameters.items(): if field not in notification_configuration: if 'default' in params: notification_configuration[field] = params['default'] backend_obj = self.notification_class(**notification_configuration) notification_obj = EmailMessage(subject, backend_obj.format_body(body), sender, recipients) with set_environ(**settings.AWX_TASK_ENV): return backend_obj.send_messages([notification_obj]) def display_notification_configuration(self): field_val = self.notification_configuration.copy() for field in self.notification_class.init_parameters: if field in field_val and force_text( field_val[field]).startswith('$encrypted$'): field_val[field] = '$encrypted$' return field_val
class WorkflowJobNode(WorkflowNodeBase): job = models.OneToOneField( 'UnifiedJob', related_name='unified_job_node', blank=True, null=True, default=None, on_delete=models.SET_NULL, ) workflow_job = models.ForeignKey( 'WorkflowJob', related_name='workflow_job_nodes', blank=True, null=True, default=None, on_delete=models.CASCADE, ) ancestor_artifacts = JSONField( blank=True, default={}, editable=False, ) def get_absolute_url(self, request=None): return reverse('api:workflow_job_node_detail', kwargs={'pk': self.pk}, request=request) def get_job_kwargs(self): ''' In advance of creating a new unified job as part of a workflow, this method builds the attributes to use It alters the node by saving its updated version of ancestor_artifacts, making it available to subsequent nodes. ''' # reject/accept prompted fields data = {} ujt_obj = self.unified_job_template if ujt_obj is not None: # MERGE note: move this to prompts_dict method on node when merging # with the workflow inventory branch prompts_data = self.prompts_dict() if isinstance(ujt_obj, WorkflowJobTemplate): if self.workflow_job.extra_vars: prompts_data.setdefault('extra_vars', {}) prompts_data['extra_vars'].update( self.workflow_job.extra_vars_dict) accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs( **prompts_data) if errors: logger.info( _('Bad launch configuration starting template {template_pk} as part of ' 'workflow {workflow_pk}. Errors:\n{error_text}').format( template_pk=ujt_obj.pk, workflow_pk=self.pk, error_text=errors)) data.update( accepted_fields) # missing fields are handled in the scheduler try: # config saved on the workflow job itself wj_config = self.workflow_job.launch_config except ObjectDoesNotExist: wj_config = None if wj_config: accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs( **wj_config.prompts_dict()) accepted_fields.pop( 'extra_vars', None) # merge handled with other extra_vars later data.update(accepted_fields) # build ancestor artifacts, save them to node model for later aa_dict = {} is_root_node = True for parent_node in self.get_parent_nodes(): is_root_node = False aa_dict.update(parent_node.ancestor_artifacts) if parent_node.job and hasattr(parent_node.job, 'artifacts'): aa_dict.update(parent_node.job.artifacts) if aa_dict and not is_root_node: self.ancestor_artifacts = aa_dict self.save(update_fields=['ancestor_artifacts']) # process password list password_dict = {} if '_ansible_no_log' in aa_dict: for key in aa_dict: if key != '_ansible_no_log': password_dict[key] = REPLACE_STR if self.workflow_job.survey_passwords: password_dict.update(self.workflow_job.survey_passwords) if self.survey_passwords: password_dict.update(self.survey_passwords) if password_dict: data['survey_passwords'] = password_dict # process extra_vars extra_vars = data.get('extra_vars', {}) if aa_dict: functional_aa_dict = copy(aa_dict) functional_aa_dict.pop('_ansible_no_log', None) extra_vars.update(functional_aa_dict) # Workflow Job extra_vars higher precedence than ancestor artifacts if ujt_obj and isinstance(ujt_obj, JobTemplate): if self.workflow_job and self.workflow_job.extra_vars: extra_vars.update(self.workflow_job.extra_vars_dict) if extra_vars: data['extra_vars'] = extra_vars # ensure that unified jobs created by WorkflowJobs are marked data['_eager_fields'] = {'launch_type': 'workflow'} # Extra processing in the case that this is a slice job if 'job_slice' in self.ancestor_artifacts and is_root_node: data['_eager_fields']['allow_simultaneous'] = True data['_eager_fields'][ 'job_slice_number'] = self.ancestor_artifacts['job_slice'] data['_eager_fields'][ 'job_slice_count'] = self.workflow_job.workflow_job_nodes.count( ) data['_prevent_slicing'] = True return data
class BaseCommandEvent(CreatedModifiedModel): """ An event/message logged from a command for each host. """ VALID_KEYS = [ 'event_data', 'created', 'counter', 'uuid', 'stdout', 'start_line', 'end_line', 'verbosity' ] class Meta: abstract = True event_data = JSONField( blank=True, default=dict, ) uuid = models.CharField( max_length=1024, default='', editable=False, ) counter = models.PositiveIntegerField( default=0, editable=False, ) stdout = models.TextField( default='', editable=False, ) verbosity = models.PositiveIntegerField( default=0, editable=False, ) start_line = models.PositiveIntegerField( default=0, editable=False, ) end_line = models.PositiveIntegerField( default=0, editable=False, ) created = models.DateTimeField( null=True, default=None, editable=False, ) modified = models.DateTimeField( default=None, editable=False, db_index=True, ) def __str__(self): return u'%s @ %s' % (self.get_event_display(), self.created.isoformat()) @classmethod def create_from_data(cls, **kwargs): # # ⚠️ D-D-D-DANGER ZONE ⚠️ # This function is called by the callback receiver *once* for *every # event* emitted by Ansible as a playbook runs. That means that # changes to this function are _very_ susceptible to introducing # performance regressions (which the user will experience as "my # playbook stdout takes too long to show up"), *especially* code which # might invoke additional database queries per event. # # Proceed with caution! # # Convert the datetime for the event's creation # appropriately, and include a time zone for it. # # In the event of any issue, throw it out, and Django will just save # the current time. try: if not isinstance(kwargs['created'], datetime.datetime): kwargs['created'] = parse_datetime(kwargs['created']) if not kwargs['created'].tzinfo: kwargs['created'] = kwargs['created'].replace(tzinfo=utc) except (KeyError, ValueError): kwargs.pop('created', None) sanitize_event_keys(kwargs, cls.VALID_KEYS) kwargs.pop('workflow_job_id', None) event = cls(**kwargs) event._update_from_event_data() return event def get_event_display(self): """ Needed for __unicode__ """ return self.event def get_event_display2(self): return self.get_event_display() def get_host_status_counts(self): return create_host_status_counts(getattr(self, 'event_data', {})) def _update_from_event_data(self): pass
class LaunchTimeConfigBase(BaseModel): ''' Needed as separate class from LaunchTimeConfig because some models use `extra_data` and some use `extra_vars`. We cannot change the API, so we force fake it in the model definitions - model defines extra_vars - use this class - model needs to use extra data - use LaunchTimeConfig Use this for models which are SurveyMixins and UnifiedJobs or Templates ''' class Meta: abstract = True # Prompting-related fields that have to be handled as special cases inventory = models.ForeignKey( 'Inventory', related_name='%(class)ss', blank=True, null=True, default=None, on_delete=models.SET_NULL, ) # All standard fields are stored in this dictionary field # This is a solution to the nullable CharField problem, specific to prompting char_prompts = JSONField(blank=True, default={}) def prompts_dict(self, display=False): data = {} # Some types may have different prompts, but always subset of JT prompts for prompt_name in JobTemplate.get_ask_mapping().keys(): try: field = self._meta.get_field(prompt_name) except FieldDoesNotExist: field = None if isinstance(field, models.ManyToManyField): if not self.pk: continue # unsaved object can't have related many-to-many prompt_val = set(getattr(self, prompt_name).all()) if len(prompt_val) > 0: data[prompt_name] = prompt_val elif prompt_name == 'extra_vars': if self.extra_vars: if display: data[prompt_name] = self.display_extra_vars() else: data[prompt_name] = self.extra_vars if self.survey_passwords and not display: data['survey_passwords'] = self.survey_passwords else: prompt_val = getattr(self, prompt_name) if prompt_val is not None: data[prompt_name] = prompt_val return data def display_extra_vars(self): ''' Hides fields marked as passwords in survey. ''' if self.survey_passwords: extra_vars = parse_yaml_or_json(self.extra_vars).copy() for key, value in self.survey_passwords.items(): if key in extra_vars: extra_vars[key] = value return extra_vars else: return self.extra_vars def display_extra_data(self): return self.display_extra_vars() @property def _credential(self): ''' Only used for workflow nodes to support backward compatibility. ''' try: return [ cred for cred in self.credentials.all() if cred.credential_type.kind == 'ssh' ][0] except IndexError: return None @property def credential(self): ''' Returns an integer so it can be used as IntegerField in serializer ''' cred = self._credential if cred is not None: return cred.pk else: return None
class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin): """A model representing a Queue/Group of AWX Instances.""" objects = InstanceGroupManager() name = models.CharField(max_length=250, unique=True) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) instances = models.ManyToManyField( 'Instance', related_name='rampart_groups', editable=False, help_text=_('Instances that are members of this InstanceGroup'), ) is_container_group = models.BooleanField(default=False) credential = models.ForeignKey( 'Credential', related_name='%(class)ss', blank=True, null=True, default=None, on_delete=models.SET_NULL, ) pod_spec_override = prevent_search( models.TextField( blank=True, default='', )) policy_instance_percentage = models.IntegerField( default=0, help_text=_( "Percentage of Instances to automatically assign to this group")) policy_instance_minimum = models.IntegerField( default=0, help_text= _("Static minimum number of Instances to automatically assign to this group" )) policy_instance_list = JSONField( default=[], blank=True, help_text= _("List of exact-match Instances that will always be automatically assigned to this group" )) POLICY_FIELDS = frozenset( ('policy_instance_list', 'policy_instance_minimum', 'policy_instance_percentage')) def get_absolute_url(self, request=None): return reverse('api:instance_group_detail', kwargs={'pk': self.pk}, request=request) @property def capacity(self): return sum(inst.capacity for inst in self.instances.all()) @property def jobs_running(self): return UnifiedJob.objects.filter(status__in=('running', 'waiting'), instance_group=self).count() @property def jobs_total(self): return UnifiedJob.objects.filter(instance_group=self).count() ''' RelatedJobsMixin ''' def _get_related_jobs(self): return UnifiedJob.objects.filter(instance_group=self) class Meta: app_label = 'main' @staticmethod def fit_task_to_most_remaining_capacity_instance(task, instances): instance_most_capacity = None for i in instances: if i.node_type not in (task.capacity_type, 'hybrid'): continue if i.remaining_capacity >= task.task_impact and ( instance_most_capacity is None or i.remaining_capacity > instance_most_capacity.remaining_capacity): instance_most_capacity = i return instance_most_capacity @staticmethod def find_largest_idle_instance(instances, capacity_type='execution'): largest_instance = None for i in instances: if i.node_type not in (capacity_type, 'hybrid'): continue if i.jobs_running == 0: if largest_instance is None: largest_instance = i elif i.capacity > largest_instance.capacity: largest_instance = i return largest_instance def set_default_policy_fields(self): self.policy_instance_list = [] self.policy_instance_minimum = 0 self.policy_instance_percentage = 0
class BasePlaybookEvent(CreatedModifiedModel): ''' An event/message logged from a playbook callback for each host. ''' VALID_KEYS = [ 'event', 'event_data', 'playbook', 'play', 'role', 'task', 'created', 'counter', 'uuid', 'stdout', 'parent_uuid', 'start_line', 'end_line', 'verbosity' ] class Meta: abstract = True # Playbook events will be structured to form the following hierarchy: # - playbook_on_start (once for each playbook file) # - playbook_on_vars_prompt (for each play, but before play starts, we # currently don't handle responding to these prompts) # - playbook_on_play_start (once for each play) # - playbook_on_import_for_host (not logged, not used for v2) # - playbook_on_not_import_for_host (not logged, not used for v2) # - playbook_on_no_hosts_matched # - playbook_on_no_hosts_remaining # - playbook_on_include (only v2 - only used for handlers?) # - playbook_on_setup (not used for v2) # - runner_on* # - playbook_on_task_start (once for each task within a play) # - runner_on_failed # - runner_on_ok # - runner_on_error (not used for v2) # - runner_on_skipped # - runner_on_unreachable # - runner_on_no_hosts (not used for v2) # - runner_on_async_poll (not used for v2) # - runner_on_async_ok (not used for v2) # - runner_on_async_failed (not used for v2) # - runner_on_file_diff (v2 event is v2_on_file_diff) # - runner_item_on_ok (v2 only) # - runner_item_on_failed (v2 only) # - runner_item_on_skipped (v2 only) # - runner_retry (v2 only) # - playbook_on_notify (once for each notification from the play, not used for v2) # - playbook_on_stats EVENT_TYPES = [ # (level, event, verbose name, failed) (3, 'runner_on_failed', _('Host Failed'), True), (3, 'runner_on_ok', _('Host OK'), False), (3, 'runner_on_error', _('Host Failure'), True), (3, 'runner_on_skipped', _('Host Skipped'), False), (3, 'runner_on_unreachable', _('Host Unreachable'), True), (3, 'runner_on_no_hosts', _('No Hosts Remaining'), False), (3, 'runner_on_async_poll', _('Host Polling'), False), (3, 'runner_on_async_ok', _('Host Async OK'), False), (3, 'runner_on_async_failed', _('Host Async Failure'), True), (3, 'runner_item_on_ok', _('Item OK'), False), (3, 'runner_item_on_failed', _('Item Failed'), True), (3, 'runner_item_on_skipped', _('Item Skipped'), False), (3, 'runner_retry', _('Host Retry'), False), # Tower does not yet support --diff mode. (3, 'runner_on_file_diff', _('File Difference'), False), (0, 'playbook_on_start', _('Playbook Started'), False), (2, 'playbook_on_notify', _('Running Handlers'), False), (2, 'playbook_on_include', _('Including File'), False), (2, 'playbook_on_no_hosts_matched', _('No Hosts Matched'), False), (2, 'playbook_on_no_hosts_remaining', _('No Hosts Remaining'), False), (2, 'playbook_on_task_start', _('Task Started'), False), # Tower does not yet support vars_prompt (and will probably hang :) (1, 'playbook_on_vars_prompt', _('Variables Prompted'), False), (2, 'playbook_on_setup', _('Gathering Facts'), False), (2, 'playbook_on_import_for_host', _('internal: on Import for Host'), False), (2, 'playbook_on_not_import_for_host', _('internal: on Not Import for Host'), False), (1, 'playbook_on_play_start', _('Play Started'), False), (1, 'playbook_on_stats', _('Playbook Complete'), False), # Additional event types for captured stdout not directly related to # playbook or runner events. (0, 'debug', _('Debug'), False), (0, 'verbose', _('Verbose'), False), (0, 'deprecated', _('Deprecated'), False), (0, 'warning', _('Warning'), False), (0, 'system_warning', _('System Warning'), False), (0, 'error', _('Error'), True), ] FAILED_EVENTS = [x[1] for x in EVENT_TYPES if x[3]] EVENT_CHOICES = [(x[1], x[2]) for x in EVENT_TYPES] LEVEL_FOR_EVENT = dict([(x[1], x[0]) for x in EVENT_TYPES]) event = models.CharField( max_length=100, choices=EVENT_CHOICES, ) event_data = JSONField( blank=True, default=dict, ) failed = models.BooleanField( default=False, editable=False, ) changed = models.BooleanField( default=False, editable=False, ) uuid = models.CharField( max_length=1024, default='', editable=False, ) playbook = models.CharField( max_length=1024, default='', editable=False, ) play = models.CharField( max_length=1024, default='', editable=False, ) role = models.CharField( max_length=1024, default='', editable=False, ) task = models.CharField( max_length=1024, default='', editable=False, ) counter = models.PositiveIntegerField( default=0, editable=False, ) stdout = models.TextField( default='', editable=False, ) verbosity = models.PositiveIntegerField( default=0, editable=False, ) start_line = models.PositiveIntegerField( default=0, editable=False, ) end_line = models.PositiveIntegerField( default=0, editable=False, ) created = models.DateTimeField( null=True, default=None, editable=False, db_index=True, ) @property def event_level(self): return self.LEVEL_FOR_EVENT.get(self.event, 0) def get_host_status_counts(self): return create_host_status_counts(getattr(self, 'event_data', {})) def get_event_display2(self): msg = self.get_event_display() if self.event == 'playbook_on_play_start': if self.play: msg = "%s (%s)" % (msg, self.play) elif self.event == 'playbook_on_task_start': if self.task: if self.event_data.get('is_conditional', False): msg = 'Handler Notified' if self.role: msg = '%s (%s | %s)' % (msg, self.role, self.task) else: msg = "%s (%s)" % (msg, self.task) # Change display for runner events triggered by async polling. Some of # these events may not show in most cases, due to filterting them out # of the job event queryset returned to the user. res = self.event_data.get('res', {}) # Fix for existing records before we had added the workaround on save # to change async_ok to async_failed. if self.event == 'runner_on_async_ok': try: if res.get('failed', False) or res.get('rc', 0) != 0: msg = 'Host Async Failed' except (AttributeError, TypeError): pass # Runner events with ansible_job_id are part of async starting/polling. if self.event in ('runner_on_ok', 'runner_on_failed'): try: module_name = res['invocation']['module_name'] job_id = res['ansible_job_id'] except (TypeError, KeyError, AttributeError): module_name = None job_id = None if module_name and job_id: if module_name == 'async_status': msg = 'Host Async Checking' else: msg = 'Host Async Started' # Handle both 1.2 on_failed and 1.3+ on_async_failed events when an # async task times out. if self.event in ('runner_on_failed', 'runner_on_async_failed'): try: if res['msg'] == 'timed out': msg = 'Host Async Timeout' except (TypeError, KeyError, AttributeError): pass return msg def _update_from_event_data(self): # Update event model fields from event data. updated_fields = set() event_data = self.event_data res = event_data.get('res', None) if self.event in self.FAILED_EVENTS and not event_data.get( 'ignore_errors', False): self.failed = True updated_fields.add('failed') if isinstance(res, dict): if res.get('changed', False): self.changed = True updated_fields.add('changed') if self.event == 'playbook_on_stats': try: failures_dict = event_data.get('failures', {}) dark_dict = event_data.get('dark', {}) self.failed = bool( sum(failures_dict.values()) + sum(dark_dict.values())) updated_fields.add('failed') changed_dict = event_data.get('changed', {}) self.changed = bool(sum(changed_dict.values())) updated_fields.add('changed') except (AttributeError, TypeError): pass for field in ('playbook', 'play', 'task', 'role'): value = force_text(event_data.get(field, '')).strip() if value != getattr(self, field): setattr(self, field, value) updated_fields.add(field) return updated_fields @classmethod def create_from_data(cls, **kwargs): pk = None for key in ('job_id', 'project_update_id'): if key in kwargs: pk = key if pk is None: # payload must contain either a job_id or a project_update_id return # Convert the datetime for the job event's creation appropriately, # and include a time zone for it. # # In the event of any issue, throw it out, and Django will just save # the current time. try: if not isinstance(kwargs['created'], datetime.datetime): kwargs['created'] = parse_datetime(kwargs['created']) if not kwargs['created'].tzinfo: kwargs['created'] = kwargs['created'].replace(tzinfo=utc) except (KeyError, ValueError): kwargs.pop('created', None) sanitize_event_keys(kwargs, cls.VALID_KEYS) job_event = cls.objects.create(**kwargs) analytics_logger.info( 'Event data saved.', extra=dict(python_objects=dict(job_event=job_event))) return job_event @property def job_verbosity(self): return 0 def save(self, *args, **kwargs): # If update_fields has been specified, add our field names to it, # if it hasn't been specified, then we're just doing a normal save. update_fields = kwargs.get('update_fields', []) # Update model fields and related objects unless we're only updating # failed/changed flags triggered from a child event. from_parent_update = kwargs.pop('from_parent_update', False) if not from_parent_update: # Update model fields from event data. updated_fields = self._update_from_event_data() for field in updated_fields: if field not in update_fields: update_fields.append(field) # Update host related field from host_name. if hasattr(self, 'job') and not self.host_id and self.host_name: if self.job.inventory.kind == 'smart': # optimization to avoid calling inventory.hosts, which # can take a long time to run under some circumstances from awx.main.models.inventory import SmartInventoryMembership membership = SmartInventoryMembership.objects.filter( inventory=self.job.inventory, host__name=self.host_name).first() if membership: host_id = membership.host_id else: host_id = None else: host_qs = self.job.inventory.hosts.filter( name=self.host_name) host_id = host_qs.only('id').values_list( 'id', flat=True).first() if host_id != self.host_id: self.host_id = host_id if 'host_id' not in update_fields: update_fields.append('host_id') super(BasePlaybookEvent, self).save(*args, **kwargs) # Update related objects after this event is saved. if hasattr(self, 'job') and not from_parent_update: if getattr(settings, 'CAPTURE_JOB_EVENT_HOSTS', False): self._update_hosts() if self.parent_uuid: kwargs = {} if self.changed is True: kwargs['changed'] = True if self.failed is True: kwargs['failed'] = True if kwargs: JobEvent.objects.filter( job_id=self.job_id, uuid=self.parent_uuid).update(**kwargs) if self.event == 'playbook_on_stats': hostnames = self._hostnames() self._update_host_summary_from_stats(hostnames) try: self.job.inventory.update_computed_fields() except DatabaseError: logger.exception( 'Computed fields database error saving event {}'. format(self.pk))
class ActivityStream(models.Model): ''' Model used to describe activity stream (audit) events ''' class Meta: app_label = 'main' ordering = ('pk', ) OPERATION_CHOICES = [ ('create', _('Entity Created')), ('update', _("Entity Updated")), ('delete', _("Entity Deleted")), ('associate', _("Entity Associated with another Entity")), ('disassociate', _("Entity was Disassociated with another Entity")) ] actor = models.ForeignKey('auth.User', null=True, on_delete=models.SET_NULL, related_name='activity_stream') operation = models.CharField(max_length=13, choices=OPERATION_CHOICES) timestamp = models.DateTimeField(auto_now_add=True) changes = models.TextField(blank=True) object_relationship_type = models.TextField(blank=True) object1 = models.TextField() object2 = models.TextField() user = models.ManyToManyField("auth.User", blank=True) organization = models.ManyToManyField("Organization", blank=True) inventory = models.ManyToManyField("Inventory", blank=True) host = models.ManyToManyField("Host", blank=True) group = models.ManyToManyField("Group", blank=True) inventory_source = models.ManyToManyField("InventorySource", blank=True) inventory_update = models.ManyToManyField("InventoryUpdate", blank=True) credential = models.ManyToManyField("Credential", blank=True) credential_type = models.ManyToManyField("CredentialType", blank=True) team = models.ManyToManyField("Team", blank=True) project = models.ManyToManyField("Project", blank=True) project_update = models.ManyToManyField("ProjectUpdate", blank=True) job_template = models.ManyToManyField("JobTemplate", blank=True) job = models.ManyToManyField("Job", blank=True) workflow_job_template_node = models.ManyToManyField( "WorkflowJobTemplateNode", blank=True) workflow_job_node = models.ManyToManyField("WorkflowJobNode", blank=True) workflow_job_template = models.ManyToManyField("WorkflowJobTemplate", blank=True) workflow_job = models.ManyToManyField("WorkflowJob", blank=True) unified_job_template = models.ManyToManyField( "UnifiedJobTemplate", blank=True, related_name='activity_stream_as_unified_job_template+') unified_job = models.ManyToManyField( "UnifiedJob", blank=True, related_name='activity_stream_as_unified_job+') ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True) schedule = models.ManyToManyField("Schedule", blank=True) custom_inventory_script = models.ManyToManyField("CustomInventoryScript", blank=True) notification_template = models.ManyToManyField("NotificationTemplate", blank=True) notification = models.ManyToManyField("Notification", blank=True) label = models.ManyToManyField("Label", blank=True) role = models.ManyToManyField("Role", blank=True) instance_group = models.ManyToManyField("InstanceGroup", blank=True) setting = JSONField(blank=True) def get_absolute_url(self, request=None): return reverse('api:activity_stream_detail', kwargs={'pk': self.pk}, request=request) def save(self, *args, **kwargs): # For compatibility with Django 1.4.x, attempt to handle any calls to # save that pass update_fields. try: super(ActivityStream, self).save(*args, **kwargs) except TypeError: if 'update_fields' not in kwargs: raise kwargs.pop('update_fields') super(ActivityStream, self).save(*args, **kwargs)
class BasePlaybookEvent(CreatedModifiedModel): """ An event/message logged from a playbook callback for each host. """ VALID_KEYS = [ 'event', 'event_data', 'playbook', 'play', 'role', 'task', 'created', 'counter', 'uuid', 'stdout', 'parent_uuid', 'start_line', 'end_line', 'host_id', 'host_name', 'verbosity', ] class Meta: abstract = True # Playbook events will be structured to form the following hierarchy: # - playbook_on_start (once for each playbook file) # - playbook_on_vars_prompt (for each play, but before play starts, we # currently don't handle responding to these prompts) # - playbook_on_play_start (once for each play) # - playbook_on_import_for_host (not logged, not used for v2) # - playbook_on_not_import_for_host (not logged, not used for v2) # - playbook_on_no_hosts_matched # - playbook_on_no_hosts_remaining # - playbook_on_include (only v2 - only used for handlers?) # - playbook_on_setup (not used for v2) # - runner_on* # - playbook_on_task_start (once for each task within a play) # - runner_on_failed # - runner_on_start # - runner_on_ok # - runner_on_error (not used for v2) # - runner_on_skipped # - runner_on_unreachable # - runner_on_no_hosts (not used for v2) # - runner_on_async_poll (not used for v2) # - runner_on_async_ok (not used for v2) # - runner_on_async_failed (not used for v2) # - runner_on_file_diff (v2 event is v2_on_file_diff) # - runner_item_on_ok (v2 only) # - runner_item_on_failed (v2 only) # - runner_item_on_skipped (v2 only) # - runner_retry (v2 only) # - playbook_on_notify (once for each notification from the play, not used for v2) # - playbook_on_stats EVENT_TYPES = [ # (level, event, verbose name, failed) (3, 'runner_on_failed', _('Host Failed'), True), (3, 'runner_on_start', _('Host Started'), False), (3, 'runner_on_ok', _('Host OK'), False), (3, 'runner_on_error', _('Host Failure'), True), (3, 'runner_on_skipped', _('Host Skipped'), False), (3, 'runner_on_unreachable', _('Host Unreachable'), True), (3, 'runner_on_no_hosts', _('No Hosts Remaining'), False), (3, 'runner_on_async_poll', _('Host Polling'), False), (3, 'runner_on_async_ok', _('Host Async OK'), False), (3, 'runner_on_async_failed', _('Host Async Failure'), True), (3, 'runner_item_on_ok', _('Item OK'), False), (3, 'runner_item_on_failed', _('Item Failed'), True), (3, 'runner_item_on_skipped', _('Item Skipped'), False), (3, 'runner_retry', _('Host Retry'), False), # Tower does not yet support --diff mode. (3, 'runner_on_file_diff', _('File Difference'), False), (0, 'playbook_on_start', _('Playbook Started'), False), (2, 'playbook_on_notify', _('Running Handlers'), False), (2, 'playbook_on_include', _('Including File'), False), (2, 'playbook_on_no_hosts_matched', _('No Hosts Matched'), False), (2, 'playbook_on_no_hosts_remaining', _('No Hosts Remaining'), False), (2, 'playbook_on_task_start', _('Task Started'), False), # Tower does not yet support vars_prompt (and will probably hang :) (1, 'playbook_on_vars_prompt', _('Variables Prompted'), False), (2, 'playbook_on_setup', _('Gathering Facts'), False), (2, 'playbook_on_import_for_host', _('internal: on Import for Host'), False), (2, 'playbook_on_not_import_for_host', _('internal: on Not Import for Host'), False), (1, 'playbook_on_play_start', _('Play Started'), False), (1, 'playbook_on_stats', _('Playbook Complete'), False), # Additional event types for captured stdout not directly related to # playbook or runner events. (0, 'debug', _('Debug'), False), (0, 'verbose', _('Verbose'), False), (0, 'deprecated', _('Deprecated'), False), (0, 'warning', _('Warning'), False), (0, 'system_warning', _('System Warning'), False), (0, 'error', _('Error'), True), ] FAILED_EVENTS = [x[1] for x in EVENT_TYPES if x[3]] EVENT_CHOICES = [(x[1], x[2]) for x in EVENT_TYPES] LEVEL_FOR_EVENT = dict([(x[1], x[0]) for x in EVENT_TYPES]) event = models.CharField( max_length=100, choices=EVENT_CHOICES, ) event_data = JSONField( blank=True, default=dict, ) failed = models.BooleanField( default=False, editable=False, ) changed = models.BooleanField( default=False, editable=False, ) uuid = models.CharField( max_length=1024, default='', editable=False, ) playbook = models.CharField( max_length=1024, default='', editable=False, ) play = models.CharField( max_length=1024, default='', editable=False, ) role = models.CharField( max_length=1024, default='', editable=False, ) task = models.CharField( max_length=1024, default='', editable=False, ) counter = models.PositiveIntegerField( default=0, editable=False, ) stdout = models.TextField( default='', editable=False, ) verbosity = models.PositiveIntegerField( default=0, editable=False, ) start_line = models.PositiveIntegerField( default=0, editable=False, ) end_line = models.PositiveIntegerField( default=0, editable=False, ) created = models.DateTimeField( null=True, default=None, editable=False, ) modified = models.DateTimeField( default=None, editable=False, db_index=True, ) @property def event_level(self): return self.LEVEL_FOR_EVENT.get(self.event, 0) def get_host_status_counts(self): return create_host_status_counts(getattr(self, 'event_data', {})) def get_event_display2(self): msg = self.get_event_display() if self.event == 'playbook_on_play_start': if self.play: msg = "%s (%s)" % (msg, self.play) elif self.event == 'playbook_on_task_start': if self.task: if self.event_data.get('is_conditional', False): msg = 'Handler Notified' if self.role: msg = '%s (%s | %s)' % (msg, self.role, self.task) else: msg = "%s (%s)" % (msg, self.task) # Change display for runner events triggered by async polling. Some of # these events may not show in most cases, due to filterting them out # of the job event queryset returned to the user. res = self.event_data.get('res', {}) # Fix for existing records before we had added the workaround on save # to change async_ok to async_failed. if self.event == 'runner_on_async_ok': try: if res.get('failed', False) or res.get('rc', 0) != 0: msg = 'Host Async Failed' except (AttributeError, TypeError): pass # Runner events with ansible_job_id are part of async starting/polling. if self.event in ('runner_on_ok', 'runner_on_failed'): try: module_name = res['invocation']['module_name'] job_id = res['ansible_job_id'] except (TypeError, KeyError, AttributeError): module_name = None job_id = None if module_name and job_id: if module_name == 'async_status': msg = 'Host Async Checking' else: msg = 'Host Async Started' # Handle both 1.2 on_failed and 1.3+ on_async_failed events when an # async task times out. if self.event in ('runner_on_failed', 'runner_on_async_failed'): try: if res['msg'] == 'timed out': msg = 'Host Async Timeout' except (TypeError, KeyError, AttributeError): pass return msg def _update_from_event_data(self): # Update event model fields from event data. event_data = self.event_data res = event_data.get('res', None) if self.event in self.FAILED_EVENTS and not event_data.get( 'ignore_errors', False): self.failed = True if isinstance(res, dict): if res.get('changed', False): self.changed = True if self.event == 'playbook_on_stats': try: failures_dict = event_data.get('failures', {}) dark_dict = event_data.get('dark', {}) self.failed = bool( sum(failures_dict.values()) + sum(dark_dict.values())) changed_dict = event_data.get('changed', {}) self.changed = bool(sum(changed_dict.values())) except (AttributeError, TypeError): pass if isinstance(self, JobEvent): try: job = self.job except ObjectDoesNotExist: job = None if job: hostnames = self._hostnames() self._update_host_summary_from_stats(set(hostnames)) if job.inventory: try: job.inventory.update_computed_fields() except DatabaseError: logger.exception( 'Computed fields database error saving event {}' .format(self.pk)) # find parent links and progagate changed=T and failed=T changed = ( job.get_event_queryset().filter(changed=True).exclude( parent_uuid=None).only('parent_uuid').values_list( 'parent_uuid', flat=True).distinct()) # noqa failed = ( job.get_event_queryset().filter(failed=True).exclude( parent_uuid=None).only('parent_uuid').values_list( 'parent_uuid', flat=True).distinct()) # noqa job.get_event_queryset().filter(uuid__in=changed).update( changed=True) job.get_event_queryset().filter(uuid__in=failed).update( failed=True) # send success/failure notifications when we've finished handling the playbook_on_stats event from awx.main.tasks.system import handle_success_and_failure_notifications # circular import def _send_notifications(): handle_success_and_failure_notifications.apply_async( [job.id]) connection.on_commit(_send_notifications) for field in ('playbook', 'play', 'task', 'role'): value = force_text(event_data.get(field, '')).strip() if value != getattr(self, field): setattr(self, field, value) if settings.LOG_AGGREGATOR_ENABLED: analytics_logger.info( 'Event data saved.', extra=dict(python_objects=dict(job_event=self))) @classmethod def create_from_data(cls, **kwargs): # # ⚠️ D-D-D-DANGER ZONE ⚠️ # This function is called by the callback receiver *once* for *every # event* emitted by Ansible as a playbook runs. That means that # changes to this function are _very_ susceptible to introducing # performance regressions (which the user will experience as "my # playbook stdout takes too long to show up"), *especially* code which # might invoke additional database queries per event. # # Proceed with caution! # pk = None for key in ('job_id', 'project_update_id'): if key in kwargs: pk = key if pk is None: # payload must contain either a job_id or a project_update_id return # Convert the datetime for the job event's creation appropriately, # and include a time zone for it. # # In the event of any issue, throw it out, and Django will just save # the current time. try: if not isinstance(kwargs['created'], datetime.datetime): kwargs['created'] = parse_datetime(kwargs['created']) if not kwargs['created'].tzinfo: kwargs['created'] = kwargs['created'].replace(tzinfo=utc) except (KeyError, ValueError): kwargs.pop('created', None) # same as above, for job_created # TODO: if this approach, identical to above, works, can convert to for loop try: if not isinstance(kwargs['job_created'], datetime.datetime): kwargs['job_created'] = parse_datetime(kwargs['job_created']) if not kwargs['job_created'].tzinfo: kwargs['job_created'] = kwargs['job_created'].replace( tzinfo=utc) except (KeyError, ValueError): kwargs.pop('job_created', None) host_map = kwargs.pop('host_map', {}) sanitize_event_keys(kwargs, cls.VALID_KEYS) workflow_job_id = kwargs.pop('workflow_job_id', None) event = cls(**kwargs) if workflow_job_id: setattr(event, 'workflow_job_id', workflow_job_id) # shouldn't job_created _always_ be present? # if it's not, how could we save the event to the db? job_created = kwargs.pop('job_created', None) if job_created: setattr(event, 'job_created', job_created) setattr(event, 'host_map', host_map) event._update_from_event_data() return event @property def job_verbosity(self): return 0
class BaseCommandEvent(CreatedModifiedModel): ''' An event/message logged from a command for each host. ''' VALID_KEYS = [ 'event_data', 'created', 'counter', 'uuid', 'stdout', 'start_line', 'end_line', 'verbosity' ] class Meta: abstract = True event_data = JSONField( blank=True, default=dict, ) uuid = models.CharField( max_length=1024, default='', editable=False, ) counter = models.PositiveIntegerField( default=0, editable=False, ) stdout = models.TextField( default='', editable=False, ) verbosity = models.PositiveIntegerField( default=0, editable=False, ) start_line = models.PositiveIntegerField( default=0, editable=False, ) end_line = models.PositiveIntegerField( default=0, editable=False, ) def __str__(self): return u'%s @ %s' % (self.get_event_display(), self.created.isoformat()) @classmethod def create_from_data(cls, **kwargs): # Convert the datetime for the event's creation # appropriately, and include a time zone for it. # # In the event of any issue, throw it out, and Django will just save # the current time. try: if not isinstance(kwargs['created'], datetime.datetime): kwargs['created'] = parse_datetime(kwargs['created']) if not kwargs['created'].tzinfo: kwargs['created'] = kwargs['created'].replace(tzinfo=utc) except (KeyError, ValueError): kwargs.pop('created', None) sanitize_event_keys(kwargs, cls.VALID_KEYS) return cls.objects.create(**kwargs) def get_event_display(self): ''' Needed for __unicode__ ''' return self.event def get_host_status_counts(self): return create_host_status_counts(getattr(self, 'event_data', {}))
class WorkflowJobNode(WorkflowNodeBase): job = models.OneToOneField( 'UnifiedJob', related_name='unified_job_node', blank=True, null=True, default=None, on_delete=models.SET_NULL, ) workflow_job = models.ForeignKey( 'WorkflowJob', related_name='workflow_job_nodes', blank=True, null=True, default=None, on_delete=models.CASCADE, ) ancestor_artifacts = JSONField( blank=True, default=dict, editable=False, ) do_not_run = models.BooleanField( default=False, help_text= _("Indicates that a job will not be created when True. Workflow runtime " "semantics will mark this True if the node is in a path that will " "decidedly not be ran. A value of False means the node may not run." ), ) identifier = models.CharField( max_length=512, blank=True, # blank denotes pre-migration job nodes help_text= _('An identifier coresponding to the workflow job template node that this node was created from.' ), ) class Meta: app_label = 'main' indexes = [ models.Index(fields=["identifier", "workflow_job"]), models.Index(fields=['identifier']), ] @property def event_processing_finished(self): return True def get_absolute_url(self, request=None): return reverse('api:workflow_job_node_detail', kwargs={'pk': self.pk}, request=request) def prompts_dict(self, *args, **kwargs): r = super(WorkflowJobNode, self).prompts_dict(*args, **kwargs) # Explanation - WFJT extra_vars still break pattern, so they are not # put through prompts processing, but inventory and others are only accepted # if JT prompts for it, so it goes through this mechanism if self.workflow_job: if self.workflow_job.inventory_id: # workflow job inventory takes precedence r['inventory'] = self.workflow_job.inventory if self.workflow_job.char_prompts: r.update(self.workflow_job.char_prompts) return r def get_job_kwargs(self): """ In advance of creating a new unified job as part of a workflow, this method builds the attributes to use It alters the node by saving its updated version of ancestor_artifacts, making it available to subsequent nodes. """ # reject/accept prompted fields data = {} ujt_obj = self.unified_job_template if ujt_obj is not None: # MERGE note: move this to prompts_dict method on node when merging # with the workflow inventory branch prompts_data = self.prompts_dict() if isinstance(ujt_obj, WorkflowJobTemplate): if self.workflow_job.extra_vars: prompts_data.setdefault('extra_vars', {}) prompts_data['extra_vars'].update( self.workflow_job.extra_vars_dict) accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs( **prompts_data) if errors: logger.info( _('Bad launch configuration starting template {template_pk} as part of ' 'workflow {workflow_pk}. Errors:\n{error_text}').format( template_pk=ujt_obj.pk, workflow_pk=self.pk, error_text=errors)) data.update( accepted_fields) # missing fields are handled in the scheduler try: # config saved on the workflow job itself wj_config = self.workflow_job.launch_config except ObjectDoesNotExist: wj_config = None if wj_config: accepted_fields, ignored_fields, errors = ujt_obj._accept_or_ignore_job_kwargs( **wj_config.prompts_dict()) accepted_fields.pop( 'extra_vars', None) # merge handled with other extra_vars later data.update(accepted_fields) # build ancestor artifacts, save them to node model for later aa_dict = {} is_root_node = True for parent_node in self.get_parent_nodes(): is_root_node = False aa_dict.update(parent_node.ancestor_artifacts) if parent_node.job and hasattr(parent_node.job, 'artifacts'): aa_dict.update(parent_node.job.artifacts) if aa_dict and not is_root_node: self.ancestor_artifacts = aa_dict self.save(update_fields=['ancestor_artifacts']) # process password list password_dict = {} if '_ansible_no_log' in aa_dict: for key in aa_dict: if key != '_ansible_no_log': password_dict[key] = REPLACE_STR if self.workflow_job.survey_passwords: password_dict.update(self.workflow_job.survey_passwords) if self.survey_passwords: password_dict.update(self.survey_passwords) if password_dict: data['survey_passwords'] = password_dict # process extra_vars extra_vars = data.get('extra_vars', {}) if ujt_obj and isinstance(ujt_obj, (JobTemplate, WorkflowJobTemplate)): if aa_dict: functional_aa_dict = copy(aa_dict) functional_aa_dict.pop('_ansible_no_log', None) extra_vars.update(functional_aa_dict) if ujt_obj and isinstance(ujt_obj, JobTemplate): # Workflow Job extra_vars higher precedence than ancestor artifacts if self.workflow_job and self.workflow_job.extra_vars: extra_vars.update(self.workflow_job.extra_vars_dict) if extra_vars: data['extra_vars'] = extra_vars # ensure that unified jobs created by WorkflowJobs are marked data['_eager_fields'] = {'launch_type': 'workflow'} if self.workflow_job and self.workflow_job.created_by: data['_eager_fields']['created_by'] = self.workflow_job.created_by # Extra processing in the case that this is a slice job if 'job_slice' in self.ancestor_artifacts and is_root_node: data['_eager_fields']['allow_simultaneous'] = True data['_eager_fields'][ 'job_slice_number'] = self.ancestor_artifacts['job_slice'] data['_eager_fields'][ 'job_slice_count'] = self.workflow_job.workflow_job_nodes.count( ) data['_prevent_slicing'] = True return data
class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEnvMixin): ''' A project represents a playbook git repo that can access a set of inventories ''' SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')] FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials'] FIELDS_TO_DISCARD_AT_COPY = ['local_path'] class Meta: app_label = 'main' ordering = ('id', ) organization = models.ForeignKey( 'Organization', blank=True, null=True, on_delete=models.CASCADE, related_name='projects', ) scm_delete_on_next_update = models.BooleanField( default=False, editable=False, ) scm_update_on_launch = models.BooleanField( default=False, help_text=_( 'Update the project when a job is launched that uses the project.' ), ) scm_update_cache_timeout = models.PositiveIntegerField( default=0, blank=True, help_text=_( 'The number of seconds after the last project update ran that a new' 'project update will be launched as a job dependency.'), ) scm_revision = models.CharField( max_length=1024, blank=True, default='', editable=False, verbose_name=_('SCM Revision'), help_text=_('The last revision fetched by a project update'), ) playbook_files = JSONField( blank=True, default=[], editable=False, verbose_name=_('Playbook Files'), help_text=_('List of playbooks found in the project'), ) inventory_files = JSONField( blank=True, default=[], editable=False, verbose_name=_('Inventory Files'), help_text= _('Suggested list of content that could be Ansible inventory in the project' ), ) admin_role = ImplicitRoleField(parent_role=[ 'organization.project_admin_role', 'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ]) use_role = ImplicitRoleField(parent_role='admin_role', ) update_role = ImplicitRoleField(parent_role='admin_role', ) read_role = ImplicitRoleField(parent_role=[ 'organization.auditor_role', 'singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR, 'use_role', 'update_role', ]) @classmethod def _get_unified_job_class(cls): return ProjectUpdate @classmethod def _get_unified_job_field_names(cls): return set(f.name for f in ProjectOptions._meta.fields) | set( ['name', 'description', 'schedule']) def save(self, *args, **kwargs): new_instance = not bool(self.pk) # If update_fields has been specified, add our field names to it, # if it hasn't been specified, then we're just doing a normal save. update_fields = kwargs.get('update_fields', []) skip_update = bool(kwargs.pop('skip_update', False)) # Check if scm_type or scm_url changes. if self.pk: project_before = self.__class__.objects.get(pk=self.pk) if project_before.scm_type != self.scm_type or project_before.scm_url != self.scm_url: self.scm_delete_on_next_update = True if 'scm_delete_on_next_update' not in update_fields: update_fields.append('scm_delete_on_next_update') # Create auto-generated local path if project uses SCM. if self.pk and self.scm_type and not self.local_path.startswith('_'): slug_name = slugify(six.text_type(self.name)).replace(u'-', u'_') self.local_path = u'_%d__%s' % (int(self.pk), slug_name) if 'local_path' not in update_fields: update_fields.append('local_path') # Do the actual save. super(Project, self).save(*args, **kwargs) if new_instance: update_fields = [] # Generate local_path for SCM after initial save (so we have a PK). if self.scm_type and not self.local_path.startswith('_'): update_fields.append('local_path') if update_fields: from awx.main.signals import disable_activity_stream with disable_activity_stream(): self.save(update_fields=update_fields) # If we just created a new project with SCM, start the initial update. if new_instance and self.scm_type and not skip_update: self.update() def _get_current_status(self): if self.scm_type: if self.current_job and self.current_job.status: return self.current_job.status elif not self.last_job: return 'never updated' # inherit the child job status on failure elif self.last_job_failed: return self.last_job.status # Return the successful status else: return self.last_job.status elif not self.get_project_path(): return 'missing' else: return 'ok' def _get_last_job_run(self): if self.scm_type and self.last_job: return self.last_job.finished else: project_path = self.get_project_path() if project_path: try: mtime = os.path.getmtime(smart_str(project_path)) dt = datetime.datetime.fromtimestamp(mtime) return make_aware(dt, get_default_timezone()) except os.error: pass def _can_update(self): return bool(self.scm_type) def _update_unified_job_kwargs(self, create_kwargs, kwargs): ''' :param create_kwargs: key-worded arguments to be updated and later used for creating unified job. :type create_kwargs: dict :param kwargs: request parameters used to override unified job template fields with runtime values. :type kwargs: dict :return: modified create_kwargs. :rtype: dict ''' if self.scm_delete_on_next_update: create_kwargs['scm_delete_on_update'] = True return create_kwargs def create_project_update(self, **kwargs): return self.create_unified_job(**kwargs) @property def cache_timeout_blocked(self): if not self.last_job_run: return False if (self.last_job_run + datetime.timedelta( seconds=self.scm_update_cache_timeout)) > now(): return True return False @property def needs_update_on_launch(self): if self.scm_type and self.scm_update_on_launch: if not self.last_job_run: return True if (self.last_job_run + datetime.timedelta( seconds=self.scm_update_cache_timeout)) <= now(): return True return False @property def notification_templates(self): base_notification_templates = NotificationTemplate.objects error_notification_templates = list( base_notification_templates.filter( unifiedjobtemplate_notification_templates_for_errors=self)) success_notification_templates = list( base_notification_templates.filter( unifiedjobtemplate_notification_templates_for_success=self)) any_notification_templates = list( base_notification_templates.filter( unifiedjobtemplate_notification_templates_for_any=self)) # Get Organization NotificationTemplates if self.organization is not None: error_notification_templates = set( error_notification_templates + list( base_notification_templates.filter( organization_notification_templates_for_errors=self. organization))) success_notification_templates = set( success_notification_templates + list( base_notification_templates.filter( organization_notification_templates_for_success=self. organization))) any_notification_templates = set(any_notification_templates + list( base_notification_templates.filter( organization_notification_templates_for_any=self. organization))) return dict(error=list(error_notification_templates), success=list(success_notification_templates), any=list(any_notification_templates)) def get_absolute_url(self, request=None): return reverse('api:project_detail', kwargs={'pk': self.pk}, request=request)