def _query_coordination(self, job_id):
"""Run Query coordination.
Query coordination will poll for query job completion, aggregate data
and then spawn a callback job to update the query cached across all
workers in the environment.
:param job_id: Job Id
:type job_id: String
"""
while True:
if self.return_jobs[job_id].failed:
self.log.critical("Query job [ %s ] encountered failures.",
job_id)
return
elif all(self.return_jobs[job_id].STDOUT.values()):
break
self.log.info("Waiting for [ %s ], QUERY to complete", job_id)
time.sleep(1)
new_task = dict()
new_task["skip_cache"] = True
new_task["extend_args"] = True
new_task["verb"] = "ARG"
query_data = dict()
for k, v in self.return_jobs[job_id].STDOUT.items():
query_data[k] = json.loads(v)
new_task["args"] = {"query": query_data}
new_task["parent_async_bypass"] = True
new_task["job_id"] = utils.get_uuid()
new_task["job_sha3_224"] = utils.object_sha3_224(obj=new_task)
new_task["parent_id"] = utils.get_uuid()
new_task["parent_sha3_224"] = utils.object_sha3_224(obj=new_task)
targets = self._get_available_workers()
self.create_return_jobs(
task=new_task["job_id"],
job_item=new_task,
targets=targets,
)
for target in targets:
self.log.debug(
"Queuing QUERY ARG callback job [ %s ] for identity"
" [ %s ]",
new_task["job_id"],
target,
)
self.send_queue.put(
dict(
identity=target,
command=new_task["verb"],
data=new_task,
))
def client(self, cache, job):
"""Run query command operation.
:param cache: Caching object used to template items within a command.
:type cache: Object
:param job: Information containing the original job specification.
:type job: Dictionary
:returns: tuple
"""
args = cache.get("args")
if args:
query = args.get(job["query"])
else:
query = None
arg_job = job.copy()
query_item = arg_job.pop("query")
targets = arg_job.get("targets", list())
if self.driver.identity in targets:
if not job.get("no_wait"):
wait_job = dict(
skip_cache=True,
verb="QUERY_WAIT",
item=query_item,
query_timeout=600,
parent_async_bypass=True,
targets=[self.driver.identity],
identity=list(),
)
wait_job["job_id"] = utils.get_uuid()
wait_job["job_sha3_224"] = utils.object_sha3_224(obj=wait_job)
wait_job["parent_id"] = arg_job["parent_id"]
wait_job["parent_sha3_224"] = utils.object_sha3_224(
obj=wait_job
)
self.block_on_tasks = [wait_job]
return (json.dumps({job["query"]: query}), None, True, job["query"])
def wrapper_func(*args, **kwargs):
self = args[0]
job = kwargs["job"]
stdout_arg = job.get("stdout_arg")
stderr_arg = job.get("stderr_arg")
stdout, stderr, outcome, command = func(*args, **kwargs)
if stdout_arg or stderr_arg:
self.block_on_tasks = list()
clean_info = (stdout.decode() if stdout
and isinstance(stdout, bytes) else stdout or "")
clean_info_err = (stderr.decode() if stderr
and isinstance(stderr, bytes) else stderr or "")
arg_job = job.copy()
arg_job.pop("parent_sha3_224", None)
arg_job.pop("parent_id", None)
arg_job.pop("job_sha3_224", None)
arg_job.pop("job_id", None)
arg_job.pop("stdout_arg", None)
arg_job.pop("stderr_arg", None)
arg_job["skip_cache"] = True
arg_job["extend_args"] = True
arg_job["verb"] = "ARG"
arg_job["args"] = {}
if stdout_arg:
arg_job["args"].update({stdout_arg: clean_info.strip()})
if stderr_arg:
arg_job["args"].update({stderr_arg: clean_info_err.strip()})
arg_job["parent_async_bypass"] = True
arg_job["targets"] = [self.driver.identity]
arg_job["job_id"] = utils.get_uuid()
arg_job["job_sha3_224"] = utils.object_sha3_224(obj=arg_job)
arg_job["parent_id"] = utils.get_uuid()
arg_job["parent_sha3_224"] = utils.object_sha3_224(obj=arg_job)
self.block_on_tasks.append(arg_job)
return stdout, stderr, outcome, command
def test_object_sha3_224(self):
sha3_224 = utils.object_sha3_224(obj={"test": "value"})
self.assertEqual(
sha3_224,
"4bc695abcb557b8e893a69389488afa07fcf9b42028de30db92c887b", # noqa
)
def format_action(
self,
verb,
execute,
arg_vars=None,
targets=None,
ignore_cache=False,
restrict=None,
parent_id=None,
parent_sha3_224=None,
parent_name=None,
job_name=None,
return_raw=False,
parent_async=False,
):
"""Return a JSON encode object for task execution.
While formatting the message, the method will treat each verb as a
case and parse the underlying sub-command, formatting the information
into a dictionary.
:param verb: Action to parse.
:type verb: String
:param execute: Execution string to parse.
:type execute: String
:param targets: Target argents to send job to.
:type targets: List
:param arg_vars: Argument dictionary, used to set arguments in
dictionary format instead of string format.
:type arg_vars: Dictionary
:param ignore_cache: Instruct the entire execution to
ignore client caching.
:type ignore_cache: Boolean
:param restrict: Restrict job execution based on a provided task
SHA3_224.
:type restrict: List
:param parent_id: Set the parent UUID for execution jobs.
:type parent_id: String
:param parent_sha3_224: Set the parent sha3_224 for execution jobs.
:type parent_sha3_224: String
:param parent_name: Set the parent name for execution jobs.
:type parent_name: String
:param job_name: Set the job name.
:type job_name: String
:param return_raw: Enable a raw return from the server.
:type return_raw: Boolean
:param parent_async: Enable a parent job to run asynchronously.
:type parent_async: Boolean
:returns: String
"""
data = dict(verb=verb)
component_kwargs = dict(exec_array=execute,
data=data,
arg_vars=arg_vars)
success, transfer, component = directord.component_import(
component=verb.lower(),
job_id=parent_id,
)
if not success:
raise SystemExit(component)
setattr(component, "verb", verb)
data.update(component.server(**component_kwargs))
data["timeout"] = getattr(component.known_args, "timeout", 600)
data["run_once"] = getattr(component.known_args, "run_once", False)
data["job_sha3_224"] = utils.object_sha3_224(obj=data)
data["return_raw"] = return_raw
data["skip_cache"] = ignore_cache or getattr(component.known_args,
"skip_cache", False)
if targets:
data["targets"] = targets
if parent_async:
data["parent_async"] = parent_async
if parent_id:
data["parent_id"] = parent_id
if parent_sha3_224:
data["parent_sha3_224"] = parent_sha3_224
if parent_name:
data["parent_name"] = parent_name
if job_name:
data["job_name"] = job_name
if restrict:
data["restrict"] = restrict
if transfer:
job = {
"jobs": [
{
"WORKDIR": "/etc/directord/components"
},
{
"ADD":
"--skip-cache {} {}".format(
transfer, "/etc/directord/components/")
},
]
}
self.exec_orchestrations(
orchestrations=[job],
defined_targets=data.get("targets"),
return_raw=True,
)
return json.dumps(data)
def exec_orchestrations(
self,
orchestrations,
defined_targets=None,
restrict=None,
ignore_cache=False,
return_raw=False,
):
"""Execute orchestration jobs.
Iterates over a list of orchestartion blobs, fingerprints the jobs,
and then runs them.
:param orchestrations: List of Dictionaries which are run as
orchestartion.
:type orchestrations: List
:param defined_targets: List of targets to limit a given execution to.
This target list provides an override for
targets found within a given orchestation.
:type defined_targets: List
:param restrict: Restrict a given orchestration job to a set of
SHA3_224 job fingerprints.
:type restrict: Array
:param ignore_cache: Instruct the orchestartion job to ignore cached
executions.
:type ignore_cache: Boolean
:param return_raw: Enable a raw return from the server.
:type return_raw: Boolean
:returns: List
"""
job_to_run = list()
for orchestrate in orchestrations:
parent_sha3_224 = utils.object_sha3_224(obj=orchestrate)
parent_name = orchestrate.get("name")
parent_id = utils.get_uuid()
targets = (defined_targets or orchestrate.pop("assign", list())
or orchestrate.get("targets", list()))
force_async = getattr(self.args, "force_async", False)
if force_async:
parent_async = force_async
else:
try:
parent_async = bool(
dist_utils.strtobool(orchestrate.get("async",
"False")))
except (ValueError, AttributeError):
parent_async = bool(orchestrate.get("async", False))
for job in orchestrate["jobs"]:
arg_vars = job.pop("vars", None)
job_name = job.pop("name", None)
assign = job.pop("assign", None)
if assign and not isinstance(assign, list):
raise SyntaxError(
"Job contained an invalid assignment: {} = {}."
" Assignments must be in list format.".format(
assign, type(assign)))
key, value = next(iter(job.items()))
job_to_run.append(
dict(
verb=key,
execute=[value],
arg_vars=arg_vars,
targets=assign or targets,
restrict=restrict,
ignore_cache=ignore_cache,
parent_id=parent_id,
parent_sha3_224=parent_sha3_224,
parent_name=parent_name,
job_name=job_name,
return_raw=return_raw,
parent_async=parent_async,
))
return_data = list()
if getattr(self.args, "finger_print", False):
count = 0
for job in job_to_run:
tabulated_data = list()
formatted_job = self.format_action(**job)
item = json.loads(formatted_job)
exec_str = " ".join(job["execute"])
if len(exec_str) >= 30:
exec_str = "{execute}...".format(execute=exec_str[:27])
tabulated_data.extend([
count,
job["parent_name"] or job["parent_sha3_224"],
item["verb"],
exec_str,
job["job_name"] or item["job_sha3_224"],
])
return_data.append(tabulated_data)
count += 1
utils.print_tabulated_data(
data=return_data,
headers=["count", "parent", "verb", "exec", "job"],
)
return []
else:
for job in job_to_run:
formatted_job = self.format_action(**job)
return_data.append(
directord.send_data(socket_path=self.args.socket_path,
data=formatted_job))
return return_data
def handle_job(
self,
command,
data,
info,
):
"""Handle a job interaction.
:param command: Command
:type command: String
:param data: Job data
:type data: Dictionary
:param info: Job info
:type info: Dictionary
"""
job = json.loads(data)
job["job_id"] = job_id = job.get("job_id", utils.get_uuid())
job["job_sha3_224"] = job_sha3_224 = job.get(
"job_sha3_224", utils.object_sha3_224(job)
)
job_parent_id = job.get("parent_id")
job_parent_sha3_224 = job.get("parent_sha3_224")
self.log.debug(
"Item received: parent job UUID [ %s ],"
" parent job sha3_224 [ %s ], job UUID [ %s ],"
" job sha3_224 [ %s ]",
job_parent_id,
job_parent_sha3_224,
job_id,
job_sha3_224,
)
with utils.ClientStatus(
job_id=job_id,
command=command,
ctx=self,
) as c:
if job_parent_id and not self._parent_check(
conn=c, cache=self.cache, job=job
):
self.q_return.put(
(
None,
None,
False,
"Job omitted, parent failure",
job,
command,
0,
None,
)
)
else:
c.job_state = self.driver.job_processing
component_kwargs = dict(cache=None, job=job)
self.log.debug(
"Queuing component [ %s ], job_id [ %s ]",
command,
job_id,
)
c.info = "task queued"
self.q_processes.put(
(
component_kwargs,
command,
info,
)
)