def __init__(self,
trail_definition,
socket_file=None,
dag_file=None,
context=None,
delay=1,
timeout=0.0001,
backlog_count=1):
self.context = context
self.root_step = make_dag(trail_definition)
self.socket_file = socket_file
self.started = False
self.timeout = timeout
self.backlog_count = backlog_count
self.trail_process = None # Will point to the trail process only in threaded mode.
self.delay = delay
for step in topological_traverse(self.root_step):
step.context = self.context
if not dag_file:
self.dag_file = mktemp(prefix='autotrail.')
else:
self.dag_file = dag_file
if not socket_file:
self.socket_file = mktemp(prefix='autotrail.socket.')
else:
self.socket_file = socket_file
self.api_socket = socket_server_setup(self.socket_file,
backlog_count=backlog_count,
timeout=timeout)
def assign_sequence_numbers_to_steps(root_step, tag_name='n'):
"""Assign unique numbers to each step in a trail.
A Step is a wrapper around an action function. However, for a given trail, Steps are unique while action_functions
may not be. E.g.,
def action_function_a(context):
pass
step_a1 = Step(action_function_a)
step_a2 = Step(action_function_a)
While the same action function is being referred to by the steps, they are different objects.
We need a way to uniquely refer to a step. While the 'name' tag allows us to refer to a Step using its action
function name, it is not unique.
This function iterates over the DAG in topological order and assigns a simple tag to each step
(calling it 'n' by default).
The assignment starts at the root step being 0 and the last topological step having the highest number.
Arguments:
root_step -- A Step like object which is the starting point of the DAG.
tag_name -- The name of the tag.
Defaults to 'n'.
"""
for number, step in enumerate(topological_traverse(root_step)):
step.tags[tag_name] = number
def deserialize_trail(root_step, trail_data, state_transformation_mapping):
"""Restore the state of a trail from trail_data.
Updates the matching step with the step's attribute data obtained from the trail_data.
This is useful when 'restoring' the state of a trail from a backed-up source.
Arguments:
root_step -- The root step of the trail DAG.
trail_data -- A dictionary containg the trail data in the following form:
{
'<action function name>': {
StatusField.STATE: <State of the step as an attribute of the Step class
eg., Step.WAIT, Step.SUCCESS>,
StatusField.RETURN_VALUE: <The string representation of the return value of
the step.>,
StatusField.PROMPT_MESSAGES: <Prompt messages from the step>,
StatusField.OUTPUT_MESSAGES:<Output messages from the step>,
StatusField.INPUT_MESSAGES: <Input messages sent to the step>,
'parents': [
<Name of parent1>,
<Name of parent2>,
],
}
}
state_transformation_mapping -- A dictionary that maps one state to another. When a step with a state that is a
key in this dictionary is restored, its state will be set to the value.
E.g., with the mapping:
{
Step.BLOCK: Step.PAUSE,
}
When a step is restored, that is in the state Step.BLOCK in trail_data, it will
be put in Step.PAUSE.
Post-condition:
Topologically traverses the DAG from the given root_step and modifies the state of the Steps based on the data
in trail_data.
Raises:
MatchTrailsException -- When the trails (from root_step and trail_data) do not match.
"""
for step in topological_traverse(root_step):
if str(step) not in trail_data:
raise MatchTrailsException(
'Step: {} does not exist in trail data.'.format(str(step)))
step_data = trail_data[str(step)]
parents = step_data['parents']
step_parents = map(str, step.parents)
if sorted(parents) != sorted(step_parents):
raise MatchTrailsException(
'The parents of step: {} do not match the ones in trail data.'.
format(str(step)))
step.state = state_transformation_mapping.get(
step_data[StatusField.STATE], step_data[StatusField.STATE])
step.return_value = step_data[StatusField.RETURN_VALUE]
step.prompt_messages = step_data[StatusField.PROMPT_MESSAGES]
step.output_messages = step_data[StatusField.OUTPUT_MESSAGES]
step.input_messages = step_data[StatusField.INPUT_MESSAGES]
def get_progeny(steps):
"""Returns the strict progeny of all the vertices provided.
Strict progeny means that each vertex returned is not a child of any other vertices.
Arguments:
steps -- An iterator over Step like objects.
Returns:
A set generator of Steps -- of Each of these steps are found in the branches originating from the given steps.
"""
return {
progeny
for step in steps for progeny in topological_traverse(step)
}
def create_namespaces_for_tag_keys_and_values(root_step):
"""Creates two Names objects from all the tags in the given DAG.
To know what a Names object is, please read the documentation of the Names class.
Arguments:
root_step -- A Step like object which is the base of the DAG.
Returns:
A tuple of Names objects of the form: (<Names of keys>, <Names of values>)
If a step is named 'first_step', then, it will have a tag called 'name'.
So, at the least, the step will have the following tags:
{
'name': 'first_step',
'n' : 0,
}
Calling this function will return two Names objects as follows:
k, v = create_namespaces_for_tag_keys_and_values(first_step)
k.name -> 'name'
v.first_step -> 'first_step'
"""
# Each step has a tags attribute which is a dictionary containing all the tags used. We need to take all the keys
# and values and add them to our list of names. Adding them allows autocompletion to be used on them as well.
keys = []
values = []
for step in topological_traverse(root_step):
for key, value in iteritems(step.tags):
# Exclude the 'n' tag, which is an integer.
if key != 'n':
keys.append(key)
values.append(value)
# Remove duplicate entries as the keys in tags are likely to be repeated.
keys = set(keys)
values = set(values)
return (Names(keys), Names(values))
def test_topological_traverse(self):
# For the DAG:
# +--> vertex_1 --> vertex_4 -->+
# vertex_0 |--> vertex_2 |--> vertex_5
# +--> vertex_3 --------------->+
#
vertices = [v for v in topological_traverse(self.vertex_0)]
vertex_0_pos = vertices.index(self.vertex_0)
vertex_1_pos = vertices.index(self.vertex_1)
vertex_2_pos = vertices.index(self.vertex_2)
vertex_3_pos = vertices.index(self.vertex_3)
vertex_4_pos = vertices.index(self.vertex_4)
vertex_5_pos = vertices.index(self.vertex_5)
# Check if the correct order is preserved
self.assertLess(vertex_0_pos, vertex_1_pos)
self.assertLess(vertex_0_pos, vertex_2_pos)
self.assertLess(vertex_0_pos, vertex_3_pos)
self.assertLess(vertex_1_pos, vertex_4_pos)
self.assertLess(vertex_4_pos, vertex_5_pos)
self.assertLess(vertex_3_pos, vertex_5_pos)
def serialize_trail(root_step):
"""Serialize a trail by serializing each Step starting at root Step.
A Step is serialized by storing the attributes of each step in the following form:
{
'<action function name>': {
StatusField.STATE: <State of the step as an attribute of the Step class eg., Step.WAIT, Step.SUCCESS>,
StatusField.RETURN_VALUE: <The string representation of the return value of the step.>,
StatusField.PROMPT_MESSAGES: <Prompt messages from the step>,
StatusField.OUTPUT_MESSAGES:<Output messages from the step>,
StatusField.INPUT_MESSAGES: <Input messages sent to the step>,
'parents': [
<Name of parent1>,
<Name of parent2>,
],
}
}
Note: The names of the children need not be stored since they are redundant. We can do this because we are not
dealing with a single Step here but the entire trail. If two vertices in 2 trails have same parents but different
children, then there will be atleast 1 child in each trail that will not match thus making the trails different.
Returns:
dictionary -- The serialized trail data in the above form.
"""
trail_data = {}
for step in topological_traverse(root_step):
parents_names = [str(parent) for parent in step.parents]
trail_data[str(step)] = {
StatusField.STATE: str(step.state),
StatusField.RETURN_VALUE: str(step.return_value),
StatusField.PROMPT_MESSAGES: step.prompt_messages,
StatusField.OUTPUT_MESSAGES: step.output_messages,
StatusField.INPUT_MESSAGES: step.input_messages,
'parents': parents_names,
}
return trail_data
def trail_manager(root_step,
api_socket,
backup,
delay=5,
context=None,
done_states=DONE_STATES,
ignore_states=IGNORE_STATES,
state_transitions=STATE_TRANSITIONS):
"""Manage a trail.
This is the lowest layer of execution of a trail, a Layer 1 client that directly manages a trail.
Using this function directly requires no knowledge of the working of autotrail, but the requirements for using this
client should be fulfilled. They are detailed in the documentation below.
Arguments:
root_step -- A Step like object fulfilling the same contract of states it can be in. A trail is represented
by the a DAG starting at root_step. A DAG can be created from a list of ordered pairs of Step
objects using the make_dag function provided in this module.
api_socket -- A socket.socket object where the API is served. All API calls are recieved and responded via
this socket.
backup -- A call-back function to backup the state of the steps. This function should accept only one
parameter viz., the root_step. It will be called with every iteration of the main trail loop
to store the state of the DAG. Avoid making this a high latency function to keep the trail
responsive.
The return value of this function is ignored.
Ensure this function is exception safe as an exception here would break out of the trail
manager loop.
Keyword arguments:
delay -- The delay before each iteration of the loop, this is the delay with which the trail_manager
iterates over the steps it is keeping track of.
It is also the delay with which it checks for any API calls.
Having a long delay will make the trail less responsive to API calls.
context -- Any object that needs to be passed to the action functions as an argument when they are run.
done_states -- A list of step states that are considered to be "done". If a step is found in this state, it
can be considered to have been run.
ignore_states -- A list of step states that will be ignored. A step in these states cannot be traversed over,
i.e., all downstream steps will be out of traversal and will never be reached (case when a
step has failed etc).
state_transitions -- A mapping of step states to functions that will be called if a step is found in that state.
These functions will be called with 2 parameters - the step and context. Their return value is
ignored. These functions can produce side effects by altering the state of the step if needed.
Responsibilities of the manager include:
1) Run the API call server.
2) Iterate over the steps in a topological traversal based on the state_functions and ignorable_state_functions
data structures.
3) Invoked the call-back backup function to save the trail state.
4) Return when the API server shuts down.
"""
logging.debug('Starting trail manager.')
# Preparing a list of steps as these are frequently needed for serving the API calls, but topological traversal
# is expensive whereas, iteration over a list is not.
steps = list(topological_traverse(root_step))
# The trail_manager uses the topological_while function, which provides a way to traverse vertices in a topological
# order (guaranteeing the trail order) while allowing the trail_manager to control the flow.
# The topological traversal has the effect that a step is not acted upon, unless all its parents are done.
# The done_check and ignore_check call-backs allow us to tell the topological_while function if a step can be
# considered done or not.
done_check = lambda step: True if step.state in done_states else False
ignore_check = lambda step: True if step.state in ignore_states else False
step_iterator = topological_while(root_step, done_check, ignore_check)
while True:
# It is important to first serve API calls before working on steps because we want a user's request to
# affect any change before the trail run changes it.
to_continue = serve_socket(api_socket,
partial(handle_api_call, steps=steps))
if to_continue is False:
logging.info('API Server says NOT to continue. Shutting down.')
api_socket.shutdown(SHUT_RDWR)
break
step = next(step_iterator, None)
if step and step.state in state_transitions:
state_transitions[step.state](step, context)
backup(root_step)
sleep(delay)