def update_computed_fields(self):
changed = self.update_computed_fields_no_save()
if not changed:
return
emit_channel_notification('schedules-changed', dict(id=self.id, group_name='schedules'))
# Must save self here before calling unified_job_template computed fields
# in order for that method to be correct
# by adding modified to update fields, we avoid updating modified time
super(Schedule, self).save(update_fields=['next_run', 'dtstart', 'dtend', 'modified'])
with ignore_inventory_computed_fields():
self.unified_job_template.update_computed_fields()
def _update_host_summary_from_stats(self, hostnames):
with ignore_inventory_computed_fields():
if not self.job or not self.job.inventory:
logger.info(
'Event {} missing job or inventory, host summaries not updated'
.format(self.pk))
return
job = self.job
from awx.main.models import Host, JobHostSummary # circular import
all_hosts = Host.objects.filter(
pk__in=self.host_map.values()).only('id')
existing_host_ids = set(h.id for h in all_hosts)
summaries = dict()
for host in hostnames:
host_id = self.host_map.get(host, None)
if host_id not in existing_host_ids:
host_id = None
host_stats = {}
for stat in ('changed', 'dark', 'failures', 'ignored', 'ok',
'processed', 'rescued', 'skipped'):
try:
host_stats[stat] = self.event_data.get(stat, {}).get(
host, 0)
except AttributeError: # in case event_data[stat] isn't a dict.
pass
summary = JobHostSummary(created=now(),
modified=now(),
job_id=job.id,
host_id=host_id,
host_name=host,
**host_stats)
summary.failed = bool(summary.dark or summary.failures)
summaries[(host_id, host)] = summary
JobHostSummary.objects.bulk_create(summaries.values())
# update the last_job_id and last_job_host_summary_id
# in single queries
host_mapping = dict((summary['host_id'], summary['id'])
for summary in JobHostSummary.objects.filter(
job_id=job.id).values('id', 'host_id'))
for h in all_hosts:
h.last_job_id = job.id
if h.id in host_mapping:
h.last_job_host_summary_id = host_mapping[h.id]
Host.objects.bulk_update(
all_hosts, ['last_job_id', 'last_job_host_summary_id'])
def update_computed_fields(self):
future_rs = dateutil.rrule.rrulestr(self.rrule, forceset=True)
next_run_actual = future_rs.after(now())
self.next_run = next_run_actual
try:
self.dtstart = future_rs[0]
except IndexError:
self.dtstart = None
self.dtend = None
if 'until' in self.rrule.lower():
match_until = re.match(".*?(UNTIL\=[0-9]+T[0-9]+Z)", self.rrule)
until_date = match_until.groups()[0].split("=")[1]
self.dtend = make_aware(datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%SZ"), get_default_timezone())
if 'count' in self.rrule.lower():
self.dtend = future_rs[-1]
emit_channel_notification('schedules-changed', dict(id=self.id, group_name='schedules'))
with ignore_inventory_computed_fields():
self.unified_job_template.update_computed_fields()
def _update_host_summary_from_stats(self, hostnames):
with ignore_inventory_computed_fields():
if not self.job or not self.job.inventory:
logger.info(
'Event {} missing job or inventory, host summaries not updated'
.format(self.pk))
return
qs = self.job.inventory.hosts.filter(name__in=hostnames)
job = self.job
for host in hostnames:
host_stats = {}
for stat in ('changed', 'dark', 'failures', 'ignored', 'ok',
'processed', 'rescued', 'skipped'):
try:
host_stats[stat] = self.event_data.get(stat, {}).get(
host, 0)
except AttributeError: # in case event_data[stat] isn't a dict.
pass
if qs.filter(name=host).exists():
host_actual = qs.get(name=host)
host_summary, created = job.job_host_summaries.get_or_create(
host=host_actual,
host_name=host_actual.name,
defaults=host_stats)
else:
host_summary, created = job.job_host_summaries.get_or_create(
host_name=host, defaults=host_stats)
if not created:
update_fields = []
for stat, value in host_stats.items():
if getattr(host_summary, stat) != value:
setattr(host_summary, stat, value)
update_fields.append(stat)
if update_fields:
host_summary.save(update_fields=update_fields)
def _update_host_summary_from_stats(self, hostnames):
with ignore_inventory_computed_fields():
try:
if not self.job or not self.job.inventory:
logger.info(
'Event {} missing job or inventory, host summaries not updated'
.format(self.pk))
return
except ObjectDoesNotExist:
logger.info(
'Event {} missing job or inventory, host summaries not updated'
.format(self.pk))
return
job = self.job
from awx.main.models import Host, JobHostSummary, HostMetric # circular import
all_hosts = Host.objects.filter(
pk__in=self.host_map.values()).only('id', 'name')
existing_host_ids = set(h.id for h in all_hosts)
summaries = dict()
updated_hosts_list = list()
for host in hostnames:
updated_hosts_list.append(host.lower())
host_id = self.host_map.get(host, None)
if host_id not in existing_host_ids:
host_id = None
host_stats = {}
for stat in ('changed', 'dark', 'failures', 'ignored', 'ok',
'processed', 'rescued', 'skipped'):
try:
host_stats[stat] = self.event_data.get(stat, {}).get(
host, 0)
except AttributeError: # in case event_data[stat] isn't a dict.
pass
summary = JobHostSummary(created=now(),
modified=now(),
job_id=job.id,
host_id=host_id,
host_name=host,
**host_stats)
summary.failed = bool(summary.dark or summary.failures)
summaries[(host_id, host)] = summary
JobHostSummary.objects.bulk_create(summaries.values())
# update the last_job_id and last_job_host_summary_id
# in single queries
host_mapping = dict((summary['host_id'], summary['id'])
for summary in JobHostSummary.objects.filter(
job_id=job.id).values('id', 'host_id'))
updated_hosts = set()
for h in all_hosts:
# if the hostname *shows up* in the playbook_on_stats event
if h.name in hostnames:
h.last_job_id = job.id
updated_hosts.add(h)
if h.id in host_mapping:
h.last_job_host_summary_id = host_mapping[h.id]
updated_hosts.add(h)
Host.objects.bulk_update(
list(updated_hosts),
['last_job_id', 'last_job_host_summary_id'],
batch_size=100)
# bulk-create
current_time = now()
HostMetric.objects.bulk_create([
HostMetric(hostname=hostname, last_automation=current_time)
for hostname in updated_hosts_list
],
ignore_conflicts=True,
batch_size=100)
HostMetric.objects.filter(hostname__in=updated_hosts_list).update(
last_automation=current_time)
def perform_update(self, options, data, inventory_update):
"""Shared method for both awx-manage CLI updates and inventory updates
from the tasks system.
This saves the inventory data to the database, calling load_into_database
but also wraps that method in a host of options processing
"""
# outside of normal options, these are needed as part of programatic interface
self.inventory = inventory_update.inventory
self.inventory_source = inventory_update.inventory_source
self.inventory_update = inventory_update
# the update options, could be parser object or dict
self.overwrite = bool(options.get('overwrite', False))
self.overwrite_vars = bool(options.get('overwrite_vars', False))
self.enabled_var = options.get('enabled_var', None)
self.enabled_value = options.get('enabled_value', None)
self.group_filter = options.get('group_filter', None) or r'^.+$'
self.host_filter = options.get('host_filter', None) or r'^.+$'
self.exclude_empty_groups = bool(options.get('exclude_empty_groups', False))
self.instance_id_var = options.get('instance_id_var', None)
try:
self.group_filter_re = re.compile(self.group_filter)
except re.error:
raise CommandError('invalid regular expression for --group-filter')
try:
self.host_filter_re = re.compile(self.host_filter)
except re.error:
raise CommandError('invalid regular expression for --host-filter')
begin = time.time()
# Since perform_update can be invoked either through the awx-manage CLI
# or from the task system, we need to create a new lock at this level
# (even though inventory_import.Command.handle -- which calls
# perform_update -- has its own lock, inventory_ID_import)
with advisory_lock('inventory_{}_perform_update'.format(self.inventory.id)):
try:
self.check_license()
except PermissionDenied as e:
self.mark_license_failure(save=True)
raise e
try:
# Check the per-org host limits
self.check_org_host_limit()
except PermissionDenied as e:
self.mark_org_limits_failure(save=True)
raise e
if settings.SQL_DEBUG:
queries_before = len(connection.queries)
# Update inventory update for this command line invocation.
with ignore_inventory_computed_fields():
# TODO: move this to before perform_update
iu = self.inventory_update
if iu.status != 'running':
with transaction.atomic():
self.inventory_update.status = 'running'
self.inventory_update.save()
logger.info('Processing JSON output...')
inventory = MemInventory(group_filter_re=self.group_filter_re, host_filter_re=self.host_filter_re)
inventory = dict_to_mem_data(data, inventory=inventory)
logger.info('Loaded %d groups, %d hosts', len(inventory.all_group.all_groups), len(inventory.all_group.all_hosts))
if self.exclude_empty_groups:
inventory.delete_empty_groups()
self.all_group = inventory.all_group
if settings.DEBUG:
# depending on inventory source, this output can be
# *exceedingly* verbose - crawling a deeply nested
# inventory/group data structure and printing metadata about
# each host and its memberships
#
# it's easy for this scale of data to overwhelm pexpect,
# (and it's likely only useful for purposes of debugging the
# actual inventory import code), so only print it if we have to:
# https://github.com/ansible/ansible-tower/issues/7414#issuecomment-321615104
self.all_group.debug_tree()
with batch_role_ancestor_rebuilding():
# If using with transaction.atomic() with try ... catch,
# with transaction.atomic() must be inside the try section of the code as per Django docs
try:
# Ensure that this is managed as an atomic SQL transaction,
# and thus properly rolled back if there is an issue.
with transaction.atomic():
# Merge/overwrite inventory into database.
if settings.SQL_DEBUG:
logger.warning('loading into database...')
with ignore_inventory_computed_fields():
if getattr(settings, 'ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC', True):
self.load_into_database()
else:
with disable_activity_stream():
self.load_into_database()
if settings.SQL_DEBUG:
queries_before2 = len(connection.queries)
self.inventory.update_computed_fields()
if settings.SQL_DEBUG:
logger.warning('update computed fields took %d queries', len(connection.queries) - queries_before2)
# Check if the license is valid.
# If the license is not valid, a CommandError will be thrown,
# and inventory update will be marked as invalid.
# with transaction.atomic() will roll back the changes.
license_fail = True
self.check_license()
# Check the per-org host limits
license_fail = False
self.check_org_host_limit()
except PermissionDenied as e:
if license_fail:
self.mark_license_failure(save=True)
else:
self.mark_org_limits_failure(save=True)
raise e
if settings.SQL_DEBUG:
logger.warning('Inventory import completed for %s in %0.1fs', self.inventory_source.name, time.time() - begin)
else:
logger.info('Inventory import completed for %s in %0.1fs', self.inventory_source.name, time.time() - begin)
# If we're in debug mode, then log the queries and time
# used to do the operation.
if settings.SQL_DEBUG:
queries_this_import = connection.queries[queries_before:]
sqltime = sum(float(x['time']) for x in queries_this_import)
logger.warning('Inventory import required %d queries ' 'taking %0.3fs', len(queries_this_import), sqltime)
def handle(self, *args, **options):
# Load inventory and related objects from database.
inventory_name = options.get('inventory_name', None)
inventory_id = options.get('inventory_id', None)
if inventory_name and inventory_id:
raise CommandError('--inventory-name and --inventory-id are mutually exclusive')
elif not inventory_name and not inventory_id:
raise CommandError('--inventory-name or --inventory-id is required')
with advisory_lock('inventory_{}_import'.format(inventory_id)):
# Obtain rest of the options needed to run update
raw_source = options.get('source', None)
if not raw_source:
raise CommandError('--source is required')
verbosity = int(options.get('verbosity', 1))
self.set_logging_level(verbosity)
# Load inventory object based on name or ID.
if inventory_id:
q = dict(id=inventory_id)
else:
q = dict(name=inventory_name)
try:
inventory = Inventory.objects.get(**q)
except Inventory.DoesNotExist:
raise CommandError('Inventory with %s = %s cannot be found' % list(q.items())[0])
except Inventory.MultipleObjectsReturned:
raise CommandError('Inventory with %s = %s returned multiple results' % list(q.items())[0])
logger.info('Updating inventory %d: %s' % (inventory.pk, inventory.name))
# Create ad-hoc inventory source and inventory update objects
with ignore_inventory_computed_fields():
source = Command.get_source_absolute_path(raw_source)
inventory_source, created = InventorySource.objects.get_or_create(
inventory=inventory,
source='file',
source_path=os.path.abspath(source),
overwrite=bool(options.get('overwrite', False)),
overwrite_vars=bool(options.get('overwrite_vars', False)),
)
inventory_update = inventory_source.create_inventory_update(
_eager_fields=dict(job_args=json.dumps(sys.argv), job_env=dict(os.environ.items()), job_cwd=os.getcwd())
)
data = AnsibleInventoryLoader(source=source, verbosity=verbosity).load()
logger.debug('Finished loading from source: %s', source)
status, tb, exc = 'error', '', None
try:
self.perform_update(options, data, inventory_update)
status = 'successful'
except Exception as e:
exc = e
if isinstance(e, KeyboardInterrupt):
status = 'canceled'
else:
tb = traceback.format_exc()
with ignore_inventory_computed_fields():
inventory_update = InventoryUpdate.objects.get(pk=inventory_update.pk)
inventory_update.result_traceback = tb
inventory_update.status = status
inventory_update.save(update_fields=['status', 'result_traceback'])
inventory_source.status = status
inventory_source.save(update_fields=['status'])
if exc:
logger.error(str(exc))
if exc:
if isinstance(exc, CommandError):
sys.exit(1)
raise exc
