def test_hosts_are_counted_once(self): mock_event = JobEvent( **{ 'event': 'playbook_on_stats', 'event_data': { 'skipped': { 'localhost': 2, 'fiz': 1, }, 'ok': { 'localhost': 1, 'foo': 2, }, 'changed': { 'localhost': 1, 'bar': 3, }, 'dark': { 'localhost': 2, 'fiz': 2, }, }, }) assert mock_event.get_host_status_counts() == { 'ok': 1, 'changed': 1, 'dark': 2 }
def test_host_summary_generation_with_deleted_hosts(): hostnames = [f'Host {i}' for i in range(10)] inv = Inventory() inv.save() Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames]) j = Job(inventory=inv) j.save() host_map = dict((host.name, host.id) for host in inv.hosts.all()) # delete half of the hosts during the playbook run for h in inv.hosts.all()[:5]: h.delete() JobEvent.create_from_data( job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats', event_data={ 'ok': dict((hostname, len(hostname)) for hostname in hostnames), 'changed': {}, 'dark': {}, 'failures': {}, 'ignored': {}, 'processed': {}, 'rescued': {}, 'skipped': {}, }, host_map=host_map, ).save() ids = sorted([s.host_id or -1 for s in j.job_host_summaries.order_by('id').all()]) names = sorted([s.host_name for s in j.job_host_summaries.all()]) assert ids == [-1, -1, -1, -1, -1, 6, 7, 8, 9, 10] assert names == ['Host 0', 'Host 1', 'Host 2', 'Host 3', 'Host 4', 'Host 5', 'Host 6', 'Host 7', 'Host 8', 'Host 9']
def test_job_event_websocket_notifications(emit): j = Job(id=123) j.save() JobEvent.create_from_data(job_id=j.pk) assert len(emit.call_args_list) == 1 topic, payload = emit.call_args_list[0][0] assert topic == 'job_events-123' assert payload['job'] == 123
def setup_environment(inventory, project, machine_credential, host, notification_template, label): ''' Create old jobs and new jobs, with various other objects to hit the related fields of Jobs. This makes sure on_delete() effects are tested properly. ''' old_jobs = [] new_jobs = [] days = 10 days_str = str(days) jt = JobTemplate.objects.create(name='testjt', inventory=inventory, project=project) jt.credentials.add(machine_credential) jt_user = User.objects.create(username='******') jt.execute_role.members.add(jt_user) notification = Notification() notification.notification_template = notification_template notification.save() for i in range(3): job1 = jt.create_job() job1.created = datetime.now(tz=timezone('UTC')) job1.save() # create jobs with current time JobEvent.create_from_data(job_id=job1.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025).save() new_jobs.append(job1) job2 = jt.create_job() # create jobs 10 days ago job2.created = datetime.now(tz=timezone('UTC')) - timedelta(days=days) job2.save() job2.dependent_jobs.add(job1) JobEvent.create_from_data(job_id=job2.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025).save() old_jobs.append(job2) jt.last_job = job2 jt.current_job = job2 jt.save() host.last_job = job2 host.save() notification.unifiedjob_notifications.add(job2) label.unifiedjob_labels.add(job2) jn = WorkflowJobNode.objects.create(job=job2) jn.save() jh = JobHostSummary.objects.create(job=job2) jh.save() return (old_jobs, new_jobs, days_str)
def test_really_long_event_fields(field): with mock.patch.object(JobEvent, 'objects') as manager: JobEvent.create_from_data(**{ 'job_id': 123, field: 'X' * 4096 }) manager.create.assert_called_with(**{ 'job_id': 123, field: 'X' * 1021 + '...' })
def test_host_summary_generation_with_limit(): # Make an inventory with 10 hosts, run a playbook with a --limit # pointed at *one* host, # Verify that *only* that host has an associated JobHostSummary and that # *only* that host has an updated value for .last_job. hostnames = [f'Host {i}' for i in range(10)] inv = Inventory() inv.save() Host.objects.bulk_create([ Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames ]) j = Job(inventory=inv) j.save() # host map is a data structure that tracks a mapping of host name --> ID # for the inventory, _regardless_ of whether or not there's a limit # applied to the actual playbook run host_map = dict((host.name, host.id) for host in inv.hosts.all()) # by making the playbook_on_stats *only* include Host 1, we're emulating # the behavior of a `--limit=Host 1` matching_host = Host.objects.get(name='Host 1') JobEvent.create_from_data( job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats', event_data={ 'ok': { matching_host.name: len(matching_host.name) }, # effectively, limit=Host 1 'changed': {}, 'dark': {}, 'failures': {}, 'ignored': {}, 'processed': {}, 'rescued': {}, 'skipped': {}, }, host_map=host_map).save() # since the playbook_on_stats only references one host, # there should *only* be on JobHostSummary record (and it should # be related to the appropriate Host) assert JobHostSummary.objects.count() == 1 for h in Host.objects.all(): if h.name == 'Host 1': assert h.last_job_id == j.id assert h.last_job_host_summary_id == JobHostSummary.objects.first( ).id else: # all other hosts in the inventory should remain untouched assert h.last_job_id is None assert h.last_job_host_summary_id is None
def test_job_events_sublist_truncation(get, organization_factory, job_template_factory, truncate, expected): objs = organization_factory("org", superusers=['admin']) jt = job_template_factory("jt", organization=objs.organization, inventory='test_inv', project='test_proj').job_template job = jt.create_unified_job() JobEvent.create_from_data(job_id=job.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025, job_created=job.created).save() url = reverse('api:job_job_events_list', kwargs={'pk': job.pk}) if not truncate: url += '?no_truncate=1' response = get(url, user=objs.superusers.admin, expect=200) assert (len(response.data['results'][0]['stdout']) == 1025) == expected
def test_parent_failed(emit, event): j = Job() j.save() JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start') assert JobEvent.objects.count() == 1 for e in JobEvent.objects.all(): assert e.failed is False JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event=event) assert JobEvent.objects.count() == 2 for e in JobEvent.objects.all(): assert e.failed is True
def test_hosts_are_counted_once(self, job, mocker): mock_event = JobEvent( **{ 'event': 'playbook_on_stats', 'event_data': { 'skipped': { 'localhost': 2, 'fiz': 1, }, 'ok': { 'localhost': 1, 'foo': 2, }, 'changed': { 'localhost': 1, 'bar': 3, }, 'dark': { 'localhost': 2, 'fiz': 2, } } }) mock_qs = namedtuple('mock_qs', ['get'])( mocker.MagicMock(return_value=mock_event)) job.job_events.only = mocker.MagicMock(return_value=mock_qs) serializer = JobDetailSerializer() host_status_counts = serializer.get_host_status_counts(job) assert host_status_counts == {'ok': 1, 'changed': 1, 'dark': 2}
def test_really_long_event_fields(field): event = JobEvent.create_from_data(**{ 'job_id': 123, 'event_data': { field: 'X' * 4096 } }) assert event.event_data[field] == 'X' * 1023 + '…'
def test_host_summary_generation(): hostnames = [f'Host {i}' for i in range(100)] inv = Inventory() inv.save() Host.objects.bulk_create([ Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames ]) j = Job(inventory=inv) j.save() host_map = dict((host.name, host.id) for host in inv.hosts.all()) JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats', event_data={ 'ok': dict((hostname, len(hostname)) for hostname in hostnames), 'changed': {}, 'dark': {}, 'failures': {}, 'ignored': {}, 'processed': {}, 'rescued': {}, 'skipped': {}, }, host_map=host_map).save() assert j.job_host_summaries.count() == len(hostnames) assert sorted([s.host_name for s in j.job_host_summaries.all()]) == sorted(hostnames) for s in j.job_host_summaries.all(): assert host_map[s.host_name] == s.host_id assert s.ok == len(s.host_name) assert s.changed == 0 assert s.dark == 0 assert s.failures == 0 assert s.ignored == 0 assert s.processed == 0 assert s.rescued == 0 assert s.skipped == 0 for host in Host.objects.all(): assert host.last_job_id == j.id assert host.last_job_host_summary.host == host
def test_parent_changed(emit): j = Job() j.save() JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start') assert JobEvent.objects.count() == 1 for e in JobEvent.objects.all(): assert e.changed is False JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='runner_on_ok', event_data={'res': { 'changed': ['localhost'] }}) assert JobEvent.objects.count() == 2 for e in JobEvent.objects.all(): assert e.changed is True
def test_unicode_with_base64_ansi(sqlite_copy_expert, get, admin): job = Job() job.save() for i in range(3): JobEvent(job=job, stdout=u'オ{}\n'.format(i), start_line=i).save() url = reverse('api:job_stdout', kwargs={ 'pk': job.pk }) + '?format=json&content_encoding=base64&content_format=ansi' response = get(url, user=admin, expect=200) content = base64.b64decode(json.loads(response.content)['content']) assert content.splitlines() == ['オ%d' % i for i in range(3)]
def job_events(self, epoch): return [ JobEvent(created=epoch), JobEvent(created=epoch + timedelta(seconds=10)), JobEvent(created=epoch + timedelta(seconds=20)), JobEvent(created=epoch + timedelta(seconds=30)), JobEvent(created=epoch + timedelta(seconds=31)), JobEvent(created=epoch + timedelta(seconds=31, milliseconds=1)), JobEvent(created=epoch + timedelta(seconds=31, microseconds=1, milliseconds=1)), ]
def test_parent_changed(emit): j = Job() j.save() JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() assert JobEvent.objects.count() == 1 for e in JobEvent.objects.all(): assert e.changed is False JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='runner_on_ok', event_data={ 'res': { 'changed': ['localhost'] } }).save() # the `playbook_on_stats` event is where we update the parent changed linkage JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() events = JobEvent.objects.filter( event__in=['playbook_on_task_start', 'runner_on_ok']) assert events.count() == 2 for e in events.all(): assert e.changed is True
def test_unicode_with_base64_ansi(sqlite_copy_expert, get, admin): created = datetime.utcnow() job = Job(created=created) job.save() for i in range(3): JobEvent(job=job, stdout='オ{}\n'.format(i), start_line=i, job_created=created).save() url = reverse('api:job_stdout', kwargs={ 'pk': job.pk }) + '?format=json&content_encoding=base64' response = get(url, user=admin, expect=200) content = base64.b64decode( json.loads(smart_str(response.content))['content']) assert smart_str(content).splitlines() == ['オ%d' % i for i in range(3)]
def test_log_from_job_event_object(): job = Job(id=4) event = JobEvent(job_id=job.id) formatter = LogstashFormatter() data_for_log = formatter.reformat_data_for_log( dict(python_objects=dict(job_event=event)), kind='job_events') # Check entire body of data for any exceptions from getattr on event object for fd in data_for_log: if not isinstance(data_for_log[fd], basestring): continue assert 'Exception' not in data_for_log[fd], 'Exception delivered in data: {}'.format(data_for_log[fd]) # Verify existence of certain high-importance fields for fd in ['changed', 'uuid', 'start_line', 'end_line', 'id', 'counter', 'host_name', 'stdout']: assert fd in data_for_log assert data_for_log['job'] == 4
def test_parent_failed(emit, event): j = Job() j.save() JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save() assert JobEvent.objects.count() == 1 for e in JobEvent.objects.all(): assert e.failed is False JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event=event).save() # the `playbook_on_stats` event is where we update the parent failed linkage JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save() events = JobEvent.objects.filter(event__in=['playbook_on_task_start', event]) assert events.count() == 2 for e in events.all(): assert e.failed is True
def test_job_job_events_children_summary_is_tree(get, organization_factory, job_template_factory): ''' children_summary should return {is_tree: False} if the event structure is not tree-like ''' objs = organization_factory("org", superusers=['admin']) jt = job_template_factory("jt", organization=objs.organization, inventory='test_inv', project='test_proj').job_template job = jt.create_unified_job() url = reverse('api:job_job_events_children_summary', kwargs={'pk': job.pk}) response = get(url, user=objs.superusers.admin, expect=200) assert response.data["event_processing_finished"] == False ''' E1 E2 E3 E4 (verbose) E5 E6 <-- parent is E2, but comes after another "branch" E5 ''' JobEvent.create_from_data(job_id=job.pk, uuid='uuid1', parent_uuid='', event="playbook_on_start", counter=1, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid2', parent_uuid='uuid1', event="playbook_on_play_start", counter=2, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid3', parent_uuid='uuid2', event="playbook_on_task_start", counter=3, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid4', parent_uuid='', event='verbose', counter=4, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid5', parent_uuid='uuid1', event="playbook_on_play_start", counter=5, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid6', parent_uuid='uuid2', event="playbook_on_task_start", counter=6, stdout='a' * 1024, job_created=job.created).save() job.emitted_events = job.get_event_queryset().count() job.status = "successful" job.save() url = reverse('api:job_job_events_children_summary', kwargs={'pk': job.pk}) response = get(url, user=objs.superusers.admin, expect=200) assert response.data["children_summary"] == {} assert response.data["meta_event_nested_uuid"] == {} assert response.data["event_processing_finished"] == True assert response.data["is_tree"] == False
def make_the_data(): with disable_activity_stream(): with batch_role_ancestor_rebuilding(), disable_computed_fields(): admin, created = User.objects.get_or_create(username='******', is_superuser=True) if created: admin.is_superuser = True admin.save() admin.set_password('test') admin.save() org_admin, created = User.objects.get_or_create( username='******') if created: org_admin.set_password('test') org_admin.save() org_member, created = User.objects.get_or_create( username='******') if created: org_member.set_password('test') org_member.save() prj_admin, created = User.objects.get_or_create( username='******') if created: prj_admin.set_password('test') prj_admin.save() jt_admin, created = User.objects.get_or_create(username='******') if created: jt_admin.set_password('test') jt_admin.save() inv_admin, created = User.objects.get_or_create( username='******') if created: inv_admin.set_password('test') inv_admin.save() print('# Creating %d organizations' % n_organizations) for i in range(n_organizations): sys.stdout.write('\r%d ' % (i + 1)) sys.stdout.flush() org, _ = Organization.objects.get_or_create( name='%s Organization %d' % (prefix, i)) organizations.append(org) if i == 0: org.admin_role.members.add(org_admin) org.member_role.members.add(org_admin) org.member_role.members.add(org_member) org.member_role.members.add(prj_admin) org.member_role.members.add(jt_admin) org.member_role.members.add(inv_admin) print('') print('# Creating %d users' % n_users) org_idx = 0 for n in spread(n_users, n_organizations): for i in range(n): ids['user'] += 1 user_id = ids['user'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, organizations[org_idx].name, i + 1)) sys.stdout.flush() user, _ = User.objects.get_or_create(username='******' % (prefix, user_id)) organizations[org_idx].member_role.members.add(user) users.append(user) org_idx += 1 print('') creator_gen = yield_choice(users) for i in range(6): next(creator_gen) modifier_gen = yield_choice(users) print('# Creating %d teams' % n_teams) org_idx = 0 for n in spread(n_teams, n_organizations): org = organizations[org_idx] for i in range(n): ids['team'] += 1 team_id = ids['team'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i + 1)) sys.stdout.flush() team, _ = Team.objects.get_or_create( name='%s Team %d Org %d' % (prefix, team_id, org_idx), organization=org, defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen))) teams.append(team) org_idx += 1 print('') print('# Adding users to teams') for org in organizations: org_teams = [t for t in org.teams.all()] org_users = [u for u in org.member_role.members.all()] print(' Spreading %d users accross %d teams for %s' % (len(org_users), len(org_teams), org.name)) # Our normal spread for most users cur_user_idx = 0 cur_team_idx = 0 for n in spread(len(org_users), len(org_teams)): team = org_teams[cur_team_idx] for i in range(n): if cur_user_idx < len(org_users): user = org_users[cur_user_idx] team.member_role.members.add(user) cur_user_idx += 1 cur_team_idx += 1 # First user gets added to all teams for team in org_teams: team.member_role.members.add(org_users[0]) print('# Creating %d credentials for users' % (n_credentials - n_credentials // 2)) user_idx = 0 for n in spread(n_credentials - n_credentials // 2, n_users): user = users[user_idx] for i in range(n): ids['credential'] += 1 sys.stdout.write('\r %d ' % (ids['credential'])) sys.stdout.flush() credential_id = ids['credential'] credential, _ = Credential.objects.get_or_create( name='%s Credential %d User %d' % (prefix, credential_id, user_idx), defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen)), credential_type=CredentialType.from_v1_kind('ssh')) credential.admin_role.members.add(user) credentials.append(credential) user_idx += 1 print('') credential_gen = yield_choice(credentials) print('# Creating %d credentials for teams' % (n_credentials // 2)) team_idx = 0 starting_credential_id = ids['credential'] for n in spread(n_credentials - n_credentials // 2, n_teams): team = teams[team_idx] for i in range(n): ids['credential'] += 1 sys.stdout.write( '\r %d ' % (ids['credential'] - starting_credential_id)) sys.stdout.flush() credential_id = ids['credential'] credential, _ = Credential.objects.get_or_create( name='%s Credential %d team %d' % (prefix, credential_id, team_idx), defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen)), credential_type=CredentialType.from_v1_kind('ssh')) credential.admin_role.parents.add(team.member_role) credentials.append(credential) team_idx += 1 print('') print('# Creating %d projects' % n_projects) org_idx = 0 for n in spread(n_projects, n_organizations): org = organizations[org_idx] for i in range(n): ids['project'] += 1 project_id = ids['project'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i + 1)) sys.stdout.flush() project, _ = Project.objects.get_or_create( name='%s Project %d Org %d' % (prefix, project_id, org_idx), organization=org, defaults=dict( created_by=next(creator_gen), modified_by=next(modifier_gen), scm_url= 'https://github.com/jlaska/ansible-playbooks.git', scm_type='git', playbook_files=[ "check.yml", "debug-50.yml", "debug.yml", "debug2.yml", "debug_extra_vars.yml", "dynamic_inventory.yml", "environ_test.yml", "fail_unless.yml", "pass_unless.yml", "pause.yml", "ping-20.yml", "ping.yml", "setfact_50.yml", "vault.yml" ])) projects.append(project) if org_idx == 0 and i == 0: project.admin_role.members.add(prj_admin) org_idx += 1 print('') print('# Creating %d inventories' % n_inventories) org_idx = 0 for n in spread(n_inventories, min(n_inventories // 4 + 1, n_organizations)): org = organizations[org_idx] for i in range(n): ids['inventory'] += 1 inventory_id = ids['inventory'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i + 1)) sys.stdout.flush() inventory, _ = Inventory.objects.get_or_create( name='%s Inventory %d Org %d' % (prefix, inventory_id, org_idx), organization=org, defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen)), variables='{"ansible_connection": "local"}') inventories.append(inventory) if org_idx == 0 and i == 0: inventory.admin_role.members.add(inv_admin) org_idx += 1 print('') print('# Creating %d inventory_groups' % n_inventory_groups) inv_idx = 0 for n in spread(n_inventory_groups, n_inventories): inventory = inventories[inv_idx] parent_list = [None] * 3 for i in range(n): ids['group'] += 1 group_id = ids['group'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, inventory.name, i + 1)) sys.stdout.flush() group, _ = Group.objects.get_or_create( name='%s Group %d Inventory %d' % (prefix, group_id, inv_idx), inventory=inventory, defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen))) # Have each group have up to 3 parent groups for parent_n in range(3): if i // 4 + parent_n < len( parent_list) and parent_list[i // 4 + parent_n]: group.parents.add(parent_list[i // 4 + parent_n]) if parent_list[i // 4] is None: parent_list[i // 4] = group else: parent_list.append(group) inventory_groups.append(group) inv_idx += 1 print('') print('# Creating %d inventory_hosts' % n_inventory_hosts) group_idx = 0 for n in spread(n_inventory_hosts, n_inventory_groups): group = inventory_groups[group_idx] for i in range(n): ids['host'] += 1 host_id = ids['host'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, group.name, i + 1)) sys.stdout.flush() host, _ = Host.objects.get_or_create( name='%s.host-%06d.group-%05d.dummy' % (prefix, host_id, group_idx), inventory=group.inventory, defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen))) # Add the host to up to 3 groups host.groups.add(group) for m in range(2): if group_idx + m < len( inventory_groups ) and group.inventory.id == inventory_groups[ group_idx + m].inventory.id: host.groups.add(inventory_groups[group_idx + m]) inventory_hosts.append(host) group_idx += 1 print('') print('# Creating %d job_templates' % n_job_templates) project_idx = 0 inv_idx = 0 for n in spread(n_job_templates, n_projects): project = projects[project_idx] for i in range(n): ids['job_template'] += 1 job_template_id = ids['job_template'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, project.name, i + 1)) sys.stdout.flush() inventory = None org_inv_count = project.organization.inventories.count() if org_inv_count > 0: inventory = project.organization.inventories.all()[ inv_idx % org_inv_count] extra_kwargs = {} job_template, _ = JobTemplate.objects.get_or_create( name='%s Job Template %d Project %d' % (prefix, job_template_id, project_idx), defaults=dict(inventory=inventory, project=project, created_by=next(creator_gen), modified_by=next(modifier_gen), playbook="debug.yml", **extra_kwargs)) job_template.credentials.add(next(credential_gen)) if ids['job_template'] % 7 == 0: job_template.credentials.add(next(credential_gen)) if ids['job_template'] % 5 == 0: # formerly cloud credential job_template.credentials.add(next(credential_gen)) job_template._is_new = _ job_templates.append(job_template) inv_idx += 1 if project_idx == 0 and i == 0: job_template.admin_role.members.add(jt_admin) project_idx += 1 if n > 0: print('') print('# Creating %d Workflow Job Templates' % n_wfjts) org_idx = 0 for n in spread(n_wfjts, n_organizations): org = organizations[org_idx] for i in range(n): ids['wfjts'] += 1 wfjt_id = ids['wfjts'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i + 1)) sys.stdout.flush() wfjt, _ = WorkflowJobTemplate.objects.get_or_create( name='%s WFJT %d Org %d' % (prefix, wfjt_id, org_idx), description=bulk_data_description, organization=org, defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen))) wfjt._is_new = _ wfjts.append(wfjt) org_idx += 1 if n: print('') print('# Creating %d Workflow Job Template nodes' % n_nodes) wfjt_idx = 0 for n in spread(n_nodes, n_wfjts): wfjt = wfjts[wfjt_idx] if not wfjt._is_new: continue jt_gen = yield_choice(job_templates) inv_gen = yield_choice(inventories) cred_gen = yield_choice(credentials) parent_idx = 0 wfjt_nodes = [] for i in range(n): ids['nodes'] += 1 sys.stdout.write('\r Assigning %d to %s: %d ' % (n, wfjt.name, i + 1)) sys.stdout.flush() kwargs = dict(workflow_job_template=wfjt, unified_job_template=next(jt_gen), modified=now()) if i % 2 == 0: # only apply inventories for every other node kwargs['inventory'] = next(inv_gen) node, _ = WorkflowJobTemplateNode.objects.get_or_create( **kwargs) if i % 3 == 0: # only apply prompted credential every 3rd node node.credentials.add(next(cred_gen)) # nodes.append(node) wfjt_nodes.append(node) if i <= 3: continue parent_node = wfjt_nodes[parent_idx] if parent_node.workflow_job_template != node.workflow_job_template: raise Exception( "Programming error, associating nodes in different workflows" ) elif parent_node == node: raise Exception("error, self association") if parent_idx % 2 == 0: parent_node.always_nodes.add(node) else: if (i + 1) % 3 == 0: parent_node.failure_nodes.add(node) else: parent_node.success_nodes.add(node) parent_idx = (parent_idx + 7) % len(wfjt_nodes) wfjt_idx += 1 if n: print('') print('# Creating %d Labels' % n_labels) org_idx = 0 for n in spread(n_labels, n_organizations): org = organizations[org_idx] for i in range(n): ids['labels'] += 1 label_id = ids['labels'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i + 1)) sys.stdout.flush() label, _ = Label.objects.get_or_create( name='%sL_%do%d' % (prefix, label_id, org_idx), organization=org, defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen))) labels.append(label) org_idx += 1 if n: print('') label_gen = yield_choice(labels) print('# Adding labels to job templates') jt_idx = 0 for n in spread(n_labels * 7, n_job_templates): if n == 0: continue jt = job_templates[jt_idx] if not jt._is_new: continue print(' Giving %d labels to %s JT' % (n, jt.name)) for i in range(n): jt.labels.add(next(label_gen)) jt_idx += 1 print('# Adding labels to workflow job templates') wfjt_idx = 0 for n in spread(n_labels * 3, n_wfjts): wfjt = wfjts[wfjt_idx] if not jt._is_new: continue print(' Giving %d labels to %s WFJT' % (n, wfjt.name)) for i in range(n): wfjt.labels.add(next(label_gen)) wfjt_idx += 1 # Disable logging here, because it will mess up output format logger = logging.getLogger('awx.main') logger.propagate = False print('# Creating %d jobs' % n_jobs) group_idx = 0 job_template_idx = 0 job_i = 0 for n in spread(n_jobs, n_job_templates): job_template = job_templates[job_template_idx] for i in range(n): sys.stdout.write('\r Assigning %d to %s: %d ' % (n, job_template.name, i + 1)) sys.stdout.flush() if len(jobs) % 4 == 0: job_stat = 'failed' elif len(jobs) % 11 == 0: job_stat = 'canceled' else: job_stat = 'successful' job, _ = Job.objects.get_or_create( job_template=job_template, status=job_stat, name="%s-%d" % (job_template.name, job_i), project=job_template.project, inventory=job_template.inventory, ) for ec in job_template.credentials.all(): job.credentials.add(ec) job._is_new = _ jobs.append(job) job_i += 1 if not job._is_new: group_idx += 1 continue if i + 1 == n: job_template.last_job = job if job_template.pk % 5 == 0: job_template.current_job = job job_template.save() if job._is_new: with transaction.atomic(): if job_template.inventory: inv_groups = [ g for g in job_template.inventory.groups.all() ] if len(inv_groups): JobHostSummary.objects.bulk_create([ JobHostSummary(job=job, host=h, host_name=h.name, processed=1, created=now(), modified=now()) for h in inv_groups[ group_idx % len(inv_groups)].hosts.all()[:100] ]) group_idx += 1 job_template_idx += 1 if n: print('') print('# Creating %d job events' % n_job_events) job_idx = 0 for n in spread(n_job_events, n_jobs): job = jobs[job_idx] # Check if job already has events, for idempotence if not job._is_new: continue # Bulk create in chunks with maximum chunk size MAX_BULK_CREATE = 100 for j in range((n // MAX_BULK_CREATE) + 1): n_subgroup = MAX_BULK_CREATE if j == n / MAX_BULK_CREATE: # on final pass, create the remainder n_subgroup = n % MAX_BULK_CREATE sys.stdout.write( '\r Creating %d job events for job %d, subgroup: %d' % (n, job.id, j + 1)) sys.stdout.flush() JobEvent.objects.bulk_create([ JobEvent(created=now(), modified=now(), job=job, event='runner_on_ok') for i in range(n_subgroup) ]) job_idx += 1 if n: print('')
def test_job_job_events_children_summary(get, organization_factory, job_template_factory): objs = organization_factory("org", superusers=['admin']) jt = job_template_factory("jt", organization=objs.organization, inventory='test_inv', project='test_proj').job_template job = jt.create_unified_job() url = reverse('api:job_job_events_children_summary', kwargs={'pk': job.pk}) response = get(url, user=objs.superusers.admin, expect=200) assert response.data["event_processing_finished"] == False ''' E1 E2 E3 E4 (verbose) E5 ''' JobEvent.create_from_data(job_id=job.pk, uuid='uuid1', parent_uuid='', event="playbook_on_start", counter=1, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid2', parent_uuid='uuid1', event="playbook_on_play_start", counter=2, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid3', parent_uuid='uuid2', event="runner_on_start", counter=3, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid4', parent_uuid='', event='verbose', counter=4, stdout='a' * 1024, job_created=job.created).save() JobEvent.create_from_data(job_id=job.pk, uuid='uuid5', parent_uuid='uuid1', event="playbook_on_task_start", counter=5, stdout='a' * 1024, job_created=job.created).save() job.emitted_events = job.get_event_queryset().count() job.status = "successful" job.save() url = reverse('api:job_job_events_children_summary', kwargs={'pk': job.pk}) response = get(url, user=objs.superusers.admin, expect=200) assert response.data["children_summary"] == { 1: { "rowNumber": 0, "numChildren": 4 }, 2: { "rowNumber": 1, "numChildren": 2 } } assert response.data["meta_event_nested_uuid"] == {4: "uuid2"} assert response.data["event_processing_finished"] == True