def traffic_runner():
servers = get_servers(SessionLocal())
for server in servers:
priv_data_dir = prepare_priv_dir(server)
ansible_runner.run_async(
private_data_dir=priv_data_dir,
project_dir="ansible/project",
playbook="traffic.yml",
extravars={"host": server.ansible_name},
finished_callback=iptables_finished_handler(server),
)
def start_playbook(playbook_name, vars):
""" Initiate a playbook run """
play_uuid = str(uuid.uuid1())
settings = {"suppress_ansible_output": True}
# this should just be run_async, using 'run' hangs the root logger output
# even when backgrounded
if vars:
_thread, _runner = run_async(
private_data_dir=configuration.settings.playbooks_root_dir,
settings=settings,
# envvars=envvars,
quiet=False,
ident=play_uuid,
extravars=vars,
# inventory='localhost',
playbook=playbook_name)
else:
_thread, _runner = run_async(
private_data_dir=configuration.settings.playbooks_root_dir,
settings=settings,
# envvars=envvars,
quiet=False,
ident=play_uuid,
# inventory='localhost',
playbook=playbook_name)
# Workaround for ansible_runner logging, resetting the rootlogger level
root_logger = logging.getLogger()
root_logger.setLevel(10)
delay = 0.1
timeout = 5 / delay
ctr = 0
# Wait for the play to actually start, but apply a timeout
while _runner.status.lower() == 'unstarted':
time.sleep(delay)
ctr += 1
if ctr > timeout:
return play_uuid, "timeout"
# Start a watcher, so the termination of the playbook can be recorded in
# the log file
_t = threading.Thread(target=watcher, args=(_thread, _runner))
_t.daemon = True
_t.name = "watcher"
_t.start()
return play_uuid, _runner.status
def iptables_runner(
port_id: int,
server_id: int,
local_port: int,
remote_ip: str = None,
remote_port: int = None,
forward_type: str = None,
update_status: bool = False,
):
if not update_status:
return
server = get_server(SessionLocal(), server_id)
priv_data_dir = prepare_priv_dir(server)
if not forward_type:
args = f" delete {local_port}"
elif remote_ip and remote_port:
args = f" -t={forward_type} forward {local_port} {remote_ip} {remote_port}"
else:
args = f" list {local_port}"
extra_vars = {
"host": server.ansible_name,
"local_port": local_port,
"iptables_args": args,
}
t = ansible_runner.run_async(
private_data_dir=priv_data_dir,
project_dir="ansible/project",
playbook="iptables.yml",
extravars=extra_vars,
status_handler=lambda s, **k: forward_rule_status_handler.delay(
port_id, s, update_status),
finished_callback=iptables_finished_handler(server, True),
)
return t[1].config.artifact_dir
def gost_runner(
port_id: int,
server_id: int,
port_num: int,
gost_config: t.Dict,
remote_ip: str = None,
update_gost: bool = False,
update_status: bool = False,
):
server = get_server(SessionLocal(), server_id)
priv_data_dir = prepare_priv_dir(server)
with open(f"ansible/project/roles/gost/files/{port_id}.json", "w") as f:
f.write(json.dumps(gost_config, indent=4))
extra_vars = {
"host": server.ansible_name,
"port_id": port_id,
"local_port": port_num,
"remote_ip": remote_ip,
"update_gost": update_gost,
"update_status": update_status,
}
r = ansible_runner.run_async(
private_data_dir=priv_data_dir,
project_dir="ansible/project",
playbook="gost.yml",
extravars=extra_vars,
status_handler=lambda s, **k: gost_status_handler.delay(
port_id, s, update_status
),
finished_callback=iptables_finished_handler(server, True)
if update_status
else lambda r: None,
)
return r[1].config.artifact_dir
def test_basic_events(is_run_async=False,g_facts=False):
tdir = tempfile.mkdtemp()
inventory = "localhost ansible_connection=local"
playbook = [{'hosts': 'all', 'gather_facts': g_facts, 'tasks': [{'debug': {'msg': "test"}}]}]
if not is_run_async:
r = run(private_data_dir=tdir,
inventory=inventory,
playbook=playbook)
else:
_, r = run_async(private_data_dir=tdir,
inventory=inventory,
playbook=playbook)
event_types = [x['event'] for x in r.events]
okay_events = [x for x in filter(lambda x: 'event' in x and x['event'] == 'runner_on_ok',
r.events)]
assert event_types[0] == 'playbook_on_start'
assert "playbook_on_play_start" in event_types
assert "runner_on_ok" in event_types
assert "playbook_on_stats" in event_types
assert r.rc == 0
if not is_run_async:
assert len(okay_events) == 1
else:
assert len(okay_events) == 2
okay_event = okay_events[0]
assert "uuid" in okay_event and len(okay_event['uuid']) == 36
assert "parent_uuid" in okay_event and len(okay_event['parent_uuid']) == 36
assert "stdout" in okay_event and len(okay_event['stdout']) > 0
assert "start_line" in okay_event and int(okay_event['start_line']) > 0
assert "end_line" in okay_event and int(okay_event['end_line']) > 0
assert "event_data" in okay_event and len(okay_event['event_data']) > 0
def run_async(
server: t.Union[Server, t.Dict],
playbook: str,
extravars: t.Dict = None,
ident: str = None,
**kwargs
):
if not server:
print("Server not found!")
return
if extravars is None:
extravars = {}
if isinstance(server, dict):
priv_data_dir = prepare_priv_dir_dict(server)
extravars["host"] = server["ansible_name"]
else:
priv_data_dir = prepare_priv_dir(server)
extravars["host"] = server.ansible_name
return ansible_runner.run_async(
ident=uuid4() if ident is None else ident,
private_data_dir=priv_data_dir,
project_dir="ansible/project",
playbook=playbook,
extravars=extravars,
**kwargs
)
def start_playbook(playbook_name, vars=None, filter=None):
""" Initiate a playbook run """
play_uuid = str(uuid.uuid1())
settings = {"suppress_ansible_output": True}
local_modules = os.path.join(configuration.settings.playbooks_root_dir,
"library")
# this should just be run_async, using 'run' hangs the root logger output
# even when backgrounded
parms = {
"private_data_dir": configuration.settings.playbooks_root_dir,
"settings": settings,
"finished_callback": cb_playbook_finished,
"event_handler": cb_event_handler,
"quiet": False,
"ident": play_uuid,
# inventory='localhost',
"playbook": playbook_name
}
if os.path.exists(local_modules):
parms["envvars"] = {
"ANSIBLE_LIBRARY": local_modules
}
if vars:
parms['extravars'] = vars
limit_hosts = filter.get('limit', None)
if limit_hosts:
parms['limit'] = limit_hosts
logger.debug("clearing up old env directory")
cleanup_dir(os.path.join(configuration.settings.playbooks_root_dir,
"env"))
_thread, _runner = run_async(**parms)
# Workaround for ansible_runner logging, resetting the rootlogger level
root_logger = logging.getLogger()
root_logger.setLevel(10)
delay = 0.1
timeout = 5 / delay
ctr = 0
# Wait for the play to actually start, but apply a timeout
while _runner.status.lower() == 'unstarted':
time.sleep(delay)
ctr += 1
if ctr > timeout:
return play_uuid, "timeout"
logger.debug("Playbook {} started in {}s".format(play_uuid,
ctr * delay))
return play_uuid, _runner.status
def installRunner(self):
installer = ansible_runner.run_async(
playbook=self.playbook,
private_data_dir=self.private_data_dir,
artifact_dir=self.artifact_dir,
ident=self.ident,
quiet=self.quiet)
return installer[1]
def run_playbook(self):
self.runner_thread, self.runner = ansible_runner.run_async(private_data_dir=self.temp_dir,
playbook="playbook.yml",
quiet=True,
debug=True,
ignore_logging=True,
cancel_callback=self.cancel_callback,
finished_callback=self.finished_callback,
event_handler=self.runner_process_message)
def start_playbook(playbook_name, vars=None, filter=None):
""" Initiate a playbook run """
play_uuid = str(uuid.uuid1())
settings = {"suppress_ansible_output": True}
local_modules = os.path.join(configuration.settings.playbooks_root_dir,
"library")
# this should just be run_async, using 'run' hangs the root logger output
# even when backgrounded
parms = {
"private_data_dir": configuration.settings.playbooks_root_dir,
"settings": settings,
"finished_callback": cb_playbook_finished,
"event_handler": cb_event_handler,
"quiet": False,
"ident": play_uuid,
# inventory='localhost',
"playbook": playbook_name
}
if os.path.exists(local_modules):
parms["envvars"] = {"ANSIBLE_LIBRARY": local_modules}
if vars:
parms['extravars'] = vars
limit_hosts = filter.get('limit', None)
if limit_hosts:
parms['limit'] = limit_hosts
logger.debug("clearing up old env directory")
cleanup_dir(os.path.join(configuration.settings.playbooks_root_dir, "env"))
_thread, _runner = run_async(**parms)
# Workaround for ansible_runner logging, resetting the rootlogger level
root_logger = logging.getLogger()
root_logger.setLevel(10)
delay = 0.1
timeout = 5 / delay
ctr = 0
# Wait for the play to actually start, but apply a timeout
while _runner.status.lower() == 'unstarted':
time.sleep(delay)
ctr += 1
if ctr > timeout:
return play_uuid, "timeout"
logger.debug("Playbook {} started in {}s".format(play_uuid, ctr * delay))
return play_uuid, _runner.status
def _execute(job: Job):
thread, runner = ansible_runner.run_async(private_data_dir=str(get_src_root() / 'ansible'),
ident=f"{job.profile.uuid}-{job.user.uuid}",
playbook='playbook.yml',
roles_path='roles',
inventory=f"{job.user.uuid} ansible_host={job.user.hostname} "
f"ansible_user={job.user.username} ansible_password={job.user.password}",
verbosity=2,
extravars=job.profile.to_dict(True))
return thread, runner
def run_playbook(group_name, playbook, private_data_dir='.', inventory='data/inventory.yaml', ident='default_ident', event_handler=general_event_handler, status_handler=general_status_handler, json_mode=True):
try:
with open(playbook) as f:
o = yaml.load(f, Loader=yaml.FullLoader)
o[-1]['hosts'] = group_name
s = open(playbook, 'w')
yaml.dump(o, s)
s.close()
f.close()
except:
pass
return ansible_runner.run_async(private_data_dir=private_data_dir, inventory=inventory, ident=ident,event_handler=event_handler, status_handler=status_handler,playbook=playbook, json_mode=json_mode)
def start_ansible_playbook(self):
# We may need to purge artifacts when we start again
if os.path.exists(os.path.join(self.temp_dir, 'artifacts')):
shutil.rmtree(os.path.join(self.temp_dir, 'artifacts'))
logger.info("runner starting")
env = os.environ.copy()
env['ANSIBLE_KERNEL_STATUS_PORT'] = str(self.helper.status_socket_port)
self.runner_thread, self.runner = ansible_runner.run_async(
private_data_dir=self.temp_dir,
playbook="playbook.yml",
quiet=True,
debug=True,
ignore_logging=True,
cancel_callback=self.cancel_callback,
finished_callback=self.finished_callback,
event_handler=self.runner_process_message)
logger.info("runner started")
logger.info("Runner status: {}".format(self.runner.status))
while self.runner.status in ['unstarted', 'running', 'starting']:
logger.info("In runner loop")
try:
logger.info("getting message %s",
self.helper.pause_socket_port)
msg = self.queue.get(timeout=1)
except queue.Empty:
logger.info("Queue Empty!")
continue
logger.info(msg)
if isinstance(msg, StatusMessage):
if self.process_message(msg.message):
break
elif isinstance(msg, TaskCompletionMessage):
logger.info('msg.task_num %s tasks_counter %s', msg.task_num,
self.tasks_counter)
break
elif not self.is_ansible_alive():
logger.info("ansible is dead")
self.do_shutdown(False)
break
logger.info("Bottom of runner loop")
time.sleep(1)
logger.info("Runner state is now {}".format(self.runner.status))
self.clean_up_task_files()
logger.info("done")
def iptables_reset_runner(
server_id: int,
port_num: int,
):
server = get_server(SessionLocal(), server_id)
priv_data_dir = prepare_priv_dir(server)
extra_vars = {
"host": server.ansible_name,
"local_port": port_num,
"iptables_args": f" reset {port_num}",
}
t = ansible_runner.run_async(
private_data_dir=priv_data_dir,
project_dir="ansible/project",
playbook="iptables.yml",
extravars=extra_vars,
)
return t[1].config.artifact_dir
def run(self):
"""run"""
runner_args = {
"json_mode": True,
"quiet": True,
"event_handler": self._event_handler,
"envvars":
{k: v
for k, v in os.environ.items() if k.startswith("ANSIBLE_")},
"cancel_callback": self.runner_cancelled_callback,
"finished_callback": self.runner_finished_callback,
}
if self._ee:
inventory = [["-i", inv] for inv in self._inventory
] if self._inventory else []
inventory = list(itertools.chain.from_iterable(inventory))
add_args = {
"cli_execenv_cmd": "playbook",
"cmdline": [self._playbook] + inventory + self._cmdline,
"container_image": self._eei,
"private_data_dir": ".",
"process_isolation_executable": self._ce,
"process_isolation": True,
}
else:
add_args = {
"cmdline": " ".join(self._cmdline),
"inventory": self._inventory,
"playbook": self._playbook,
}
runner_args.update(add_args)
for key, value in runner_args.items():
self._logger.debug("Runner arg: %s:%s", key, value)
thread, _runner = run_async(**runner_args)
self.status = "running"
return thread
def tc_runner(server_id: int,
port_num: int,
egress_limit: int = None,
ingress_limit: int = None):
server = get_server(SessionLocal(), server_id)
priv_data_dir = prepare_priv_dir(server)
args = ""
if egress_limit:
args += f' -e={egress_limit}kbit'
if ingress_limit:
args += f' -i={ingress_limit}kbit'
args += f' {port_num}'
t = ansible_runner.run_async(
private_data_dir=priv_data_dir,
project_dir="ansible/project",
playbook="tc.yml",
extravars={
"host": server.ansible_name,
"tc_args": args
},
)
return t[1].config.artifact_dir
def start(self, project, playbook):
logger.debug("STARTING %s: %s, %s", str(self), project, playbook)
running = False
self.runnerconfig = ansible_runner.run_async(
private_data_dir=self.data_dir,
playbook=playbook,
event_handler=self.handle_event,
status_handler=self.handle_status,
finished_callback=self.finish,
ident=str(self.op_id),
quiet=True)
# Note: config files override some of of these
# project_dir = project,
# json_mode = True
# ident = str(self.op_id) + '_' + playbook[:-4],
logger.info(
"private_data_dir: %s, project: %s, playbook: %s, ident: %s",
self.data_dir, project, playbook, str(self.op_id))
try:
job = ansible_runner.run(config=self.runnerconfig)
logger.debug("job: %s", str(job))
self.runner = job[1]
self.stdout = self.runner.stdout
running = True
except ValueError as e:
logger.error("%s", str(e))
if self.handler: #run_handler or done_handler?
self.handler({
'error': True,
'op_id': self.op_id,
'stdout': str(e)
})
finally:
return running
def start_playbook(playbook_name, vars=None, filter=None, tags=None):
""" Initiate a playbook run """
r = APIResponse()
play_uuid = str(uuid.uuid1())
settings = {"suppress_ansible_output": True}
local_modules = os.path.join(configuration.settings.playbooks_root_dir,
"library")
# this should just be run_async, using 'run' hangs the root logger output
# even when backgrounded
parms = {
"private_data_dir": configuration.settings.playbooks_root_dir,
"settings": settings,
"finished_callback": cb_playbook_finished,
"event_handler": cb_event_handler,
"quiet": False,
"ident": play_uuid,
"playbook": playbook_name
}
if os.path.exists(local_modules):
parms["envvars"] = {"ANSIBLE_LIBRARY": local_modules}
if vars:
parms['extravars'] = vars
limit_hosts = filter.get('limit', None)
if limit_hosts:
parms['limit'] = limit_hosts
logger.debug("Clearing up old env directory")
cleanup_dir(os.path.join(configuration.settings.playbooks_root_dir, "env"))
cmdline = []
if tags:
cmdline.append("--tags {}".format(tags))
if configuration.settings.target_user != getpass.getuser():
logger.debug("Run the playbook with a user override of "
"{}".format(configuration.settings.target_user))
cmdline.append("--user {}".format(configuration.settings.target_user))
if cmdline:
commit_cmdline(cmdline)
_thread, _runner = run_async(**parms)
# Workaround for ansible_runner logging, resetting the rootlogger level
root_logger = logging.getLogger()
root_logger.setLevel(10)
delay = 0.1
timeout = 5 / delay
ctr = 0
# Wait for the play to actually start, but apply a timeout
while _runner.status.lower() == 'unstarted':
time.sleep(delay)
ctr += 1
if ctr > timeout:
r.status, r.msg = "TIMEOUT", "Timeout hit while waiting for " \
"playbook to start"
return r
logger.debug("Playbook {} started in {}s".format(play_uuid, ctr * delay))
r.status, r.data = "OK", {"status": _runner.status, "play_uuid": play_uuid}
runner_cache[play_uuid] = {
"runner": _runner,
"status": _runner.status,
"current_task": None,
"current_task_metadata": {
"created": "",
"play_pattern": "",
"task_path": "",
"task_action": ""
},
"role": "",
"last_task_num": None,
"start_epoc": time.time(),
"skipped": 0,
"failed": 0,
"ok": 0,
"failures": {}
}
# add uuid to cache so it can be filled with its events
event_cache[play_uuid] = {'time': datetime.datetime.now()}
# limit event cache size
if len(event_cache) > configuration.settings.event_cache_size:
oldest = play_uuid
for ident in event_cache:
if event_cache[ident]['time'] < event_cache[oldest]['time']:
oldest = ident
del event_cache[oldest]
return r
def provision_and_test(extravars):
# # Path is static in the container
# local_repo = '/var/www/html'
# extravars['local_repo'] = local_repo
if 'topic' in extravars.keys():
print("Topic is %s" % extravars['topic'])
else:
print("Error ! No topic found in settings.")
sys.exit(1)
# This function is kept for backward compatibility.
if 'download_only' in extravars.keys():
if extravars['download_only'] == True:
print('The dci-rhel-agent is configured in download-only mode.')
sys.exit(0)
r = ansible_runner.run(private_data_dir="/usr/share/dci-rhel-agent/",
inventory="/etc/dci-rhel-agent/inventory",
verbosity=1,
playbook="dci-import.yml",
extravars=extravars,
quiet=False)
if r.rc != 0:
print("Distro(s) import in Beaker has failed. {}: {}".format(
r.status, r.rc))
sys.exit(1)
if 'systems' not in extravars.keys():
print(
'No hosts found in settings. You should configure download-only mode or add systems[].'
)
sys.exit(1)
fqdns = extravars['systems']
threads_runners = {}
for fqdn in fqdns:
print("Starting job for %s." % fqdn)
extravars['fqdn'] = fqdn
thread, runner = ansible_runner.run_async(
private_data_dir="/usr/share/dci-rhel-agent/",
inventory="/etc/dci-rhel-agent/inventory",
verbosity=1,
playbook="dci-rhel-agent.yml",
extravars=extravars,
quiet=False)
threads_runners[(thread, runner)] = fqdn
# wait for all jobs
for t, _ in threads_runners:
t.join()
print("All jobs terminated.")
global number_of_failed_jobs
# check if some jobs failed
for t, r in threads_runners:
fqdn = threads_runners[(t, r)]
if r.rc != 0:
print("Job for %s failed, rc: %s, status: %s " %
(fqdn, r.rc, r.status))
number_of_failed_jobs += 1
def test_basic_events(containerized,
container_runtime_available,
is_pre_ansible28,
is_run_async=False,
g_facts=False):
if containerized and not container_runtime_available:
pytest.skip('container runtime(s) not available')
tdir = tempfile.mkdtemp()
if is_pre_ansible28:
inventory = 'localhost ansible_connection=local ansible_python_interpreter="/usr/bin/env python"'
else:
inventory = 'localhost ansible_connection=local'
playbook = [{
'hosts': 'all',
'gather_facts': g_facts,
'tasks': [{
'debug': {
'msg': "test"
}
}]
}]
run_args = {
'private_data_dir': tdir,
'inventory': inventory,
'envvars': {
"ANSIBLE_DEPRECATION_WARNINGS": "False",
'ANSIBLE_PYTHON_INTERPRETER': 'auto_silent'
},
'playbook': playbook
}
if containerized:
run_args.update({
'process_isolation': True,
'process_isolation_executable': 'podman',
'container_image': defaults.default_container_image,
'container_volume_mounts': [f'{tdir}:{tdir}']
})
if not is_run_async:
r = run(**run_args)
else:
thread, r = run_async(**run_args)
thread.join() # ensure async run finishes
event_types = [x['event'] for x in r.events if x['event'] != 'verbose']
okay_events = [
x for x in filter(
lambda x: 'event' in x and x['event'] == 'runner_on_ok', r.events)
]
assert event_types[0] == 'playbook_on_start'
assert "playbook_on_play_start" in event_types
assert "runner_on_ok" in event_types
assert "playbook_on_stats" in event_types
assert r.rc == 0
if not is_run_async:
assert len(okay_events) == 1
else:
assert len(okay_events) == 2
okay_event = okay_events[0]
assert "uuid" in okay_event and len(okay_event['uuid']) == 36
assert "parent_uuid" in okay_event and len(okay_event['parent_uuid']) == 36
assert "stdout" in okay_event and len(okay_event['stdout']) > 0
assert "start_line" in okay_event and int(okay_event['start_line']) > 0
assert "end_line" in okay_event and int(okay_event['end_line']) > 0
assert "event_data" in okay_event and len(okay_event['event_data']) > 0
parser = argparse.ArgumentParser()
parser.add_argument('--project', required=True)
parser.add_argument('--debug', action='store_true', default=False)
args = parser.parse_args()
if args.debug:
log.setLevel(logging.DEBUG)
data_dir = '/opt/' + args.project
playbook = args.project + '.yml'
log.info('ansible_runner startup, project: %s' % args.project)
log.info('- data_dir: %s' % data_dir)
log.info('- playbook: %s' % playbook)
r = redis.StrictRedis(host='localhost', port=6379, db=0)
p = r.pubsub()
p.subscribe('bootstrap')
signal.signal(signal.SIGINT, signal_handler)
housekeeping()
for m in p.listen():
if m['type'] == 'message':
log.debug('incoming message...')
data = json.loads(m['data'])
hostname = data['bootstrap']
task_list.append(ansible_runner.run_async(playbook=playbook,
private_data_dir=data_dir,
limit=hostname,
quiet=True))
def provision_and_test(extravars):
# Path is static in the container
# local_repo = '/var/www/html'
# extravars['local_repo'] = local_repo
if 'topic' in extravars.keys():
print("Topic is %s" % extravars['topic'])
else:
print("Error ! No topic found in settings.")
sys.exit(1)
# Provision and install SUT
if 'systems' not in extravars.keys():
print(
'No hosts found in settings. Please add systems to provision and/or test to your settings file.'
)
sys.exit(1)
# Setup conserver if a sol_command exist
if [
system for system in extravars['systems']
if type(system) is dict and 'sol_command' in system.keys()
]:
systems = {
'systems': [
system for system in extravars['systems']
if type(system) is dict and 'sol_command' in system.keys()
]
}
r = ansible_runner.run(private_data_dir="/usr/share/dci-rhel-agent/",
inventory="/etc/dci-rhel-agent/inventory",
verbosity=1,
playbook="conserver.yml",
extravars=systems,
quiet=False)
if r.rc != 0:
print("Conserver playbook failed. {}: {}".format(r.status, r.rc))
sys.exit(1)
threads_runners = {}
for system in extravars['systems']:
if type(system) is dict and 'fqdn' in system:
extravars['fqdn'] = system['fqdn']
if 'kernel_options' in system:
extravars['kernel_options'] = system['kernel_options']
else:
extravars.pop('kernel_options', None)
if 'ks_meta' in system:
extravars['ks_meta'] = system['ks_meta']
else:
extravars.pop('ks_meta', None)
if 'sol_command' in system:
extravars['sol_command'] = system['sol_command']
else:
extravars.pop('sol_command', None)
if 'sut_password' in system:
extravars['sut_password'] = system['sut_password']
else:
extravars.pop('sut_password', None)
if 'reboot_watchdog_timeout' in system:
extravars['reboot_watchdog_timeout'] = system[
'reboot_watchdog_timeout']
else:
extravars.pop('reboot_watchdog_timeout', None)
if 'install_watchdog_timeout' in system:
extravars['install_watchdog_timeout'] = system[
'install_watchdog_timeout']
else:
extravars.pop('install_watchdog_timeout', None)
else:
extravars['fqdn'] = system
#Remove any install options set for previous SUTs in this topic if they exist
extravars.pop('kernel_options', None)
extravars.pop('ks_meta', None)
extravars.pop('sol_command', None)
extravars.pop('reboot_watchdog_timeout', None)
extravars.pop('install_watchdog_timeout', None)
print("Starting job for %s." % extravars['fqdn'])
thread, runner = ansible_runner.run_async(
private_data_dir="/usr/share/dci-rhel-agent/",
inventory="/etc/dci-rhel-agent/inventory",
verbosity=int(environ.get('VERBOSITY')),
playbook="dci-rhel-agent.yml",
extravars=extravars,
quiet=False)
threads_runners[(thread, runner)] = extravars['fqdn']
# wait for all jobs
for t, _ in threads_runners:
t.join()
print("All jobs terminated.")
global number_of_failed_jobs
# check if some jobs failed
for t, r in threads_runners:
fqdn = threads_runners[(t, r)]
if r.rc != 0:
print("Job for %s failed, rc: %s, status: %s " %
(fqdn, r.rc, r.status))
number_of_failed_jobs += 1
