def test_get_actions_something():
# Arrange
widget = {'properties': {'tagName': 'SOMETHING'}}
# Act
actions = StateAbstracter.get_actions(widget)
# Assert
assert len(actions) == 0
def test_get_actions_button():
# Arrange
widget = {'properties': {'tagName': 'BUTTON'}}
# Act
actions = StateAbstracter.get_actions(widget)
# Assert
assert actions[0] == 'click'
def test_get_actions_input():
# Arrange
widget = {'properties': {'tagName': 'INPUT'}}
# Act
actions = StateAbstracter.get_actions(widget)
# Assert
assert actions[0] == 'set'
def test_process(build_selector_mock, get_actions_mock):
# Arrange
concrete_state = {
'widgets': {
'key_1': {
'key': 'key_1',
'properties': {
'tagName': 'A'
}
},
'key_2': {
'key': 'key_2',
'properties': {
'tagName': 'INPUT'
}
},
'key_3': {
'key': 'key_3',
'properties': {
'tagName': 'SOMETHING'
}
},
'key_4': {
'key': 'key_4',
'properties': {
'tagName': 'INPUT'
}
},
}
}
state_abstracter = StateAbstracter()
# Act
result = state_abstracter.process(concrete_state)
# Assert
assert len(result.widgets) == 2
assert result.widgets[0]['selector'] == 'selector_1'
assert result.widgets[1]['selector'] == 'selector_2'
assert len(result.static_widgets) == 1
assert result.static_widgets[0]['key'] == 'key_3'
assert build_selector_mock.call_count == 3
assert get_actions_mock.call_count == 3
assert result.hash != 0
def test_build_selector_widget_has_href():
# Arrange
widget = {'properties': {'tagName': 'tag_name', 'href': 'href_1'}}
selectors = {}
# Act
result = StateAbstracter.build_selector(widget, selectors)
# Assert
assert selectors["tag_name[href='href_1']"] == 0
assert result == "tag_name[href='href_1']:visible:eq(0)"
def test_build_selector_does_already_exist():
# Arrange
widget = {'properties': {'tagName': 'tag_name'}}
selectors = {'tag_name': 0}
# Act
result = StateAbstracter.build_selector(widget, selectors)
# Assert
assert selectors['tag_name'] == 1
assert result == 'tag_name:visible:eq(1)'
def __init__(self,
sut_url,
runner_url,
form_expert_client=None,
runner_client=None,
page_analysis_client=None,
flow_generator_client=None,
flow_publisher=None,
flow_executor=None):
""" Initializes the AgentLoop class.
:param sut_url: The URL of the SUT.
:param runner_url: The URL to the runner resource to be used for driving the SUT.
"""
self.sut_url = sut_url
self.runner_url = runner_url
self.mapper = StateAbstracter()
self.label_extracter = LabelExtraction()
self.form_expert = form_expert_client if form_expert_client else FormExpertClient(
)
self.memory = PriorityMemory()
self.observer = StateObserver()
self.seq_parser = SequenceParser()
self.flow_planner = FlowPlanner()
self.defect_rep = DefectReporter()
self.flow_publish = flow_publisher if flow_publisher else PlannedFlowPublisher(
)
self.runner = runner_client if runner_client else RunnerClient(
self.runner_url)
self.page_analyzer = page_analysis_client if page_analysis_client else PageAnalysisClient(
)
self.flow_generator = flow_generator_client if flow_generator_client else FlowGeneratorClient(
)
self.flow_executer = flow_executor if flow_executor else FlowExecutor(
self.form_expert, self.page_analyzer, self.mapper,
self.label_extracter, self.observer, self.defect_rep)
class AgentLoop:
""" Responsible for implementing the core control flow of the agent.
Observes the SUT environment, plans, and acts upon the environment."""
NUM_ITERATIONS = 300
def __init__(self,
sut_url,
runner_url,
form_expert_client=None,
runner_client=None,
page_analysis_client=None,
flow_generator_client=None,
flow_publisher=None,
flow_executor=None):
""" Initializes the AgentLoop class.
:param sut_url: The URL of the SUT.
:param runner_url: The URL to the runner resource to be used for driving the SUT.
"""
self.sut_url = sut_url
self.runner_url = runner_url
self.mapper = StateAbstracter()
self.label_extracter = LabelExtraction()
self.form_expert = form_expert_client if form_expert_client else FormExpertClient(
)
self.memory = PriorityMemory()
self.observer = StateObserver()
self.seq_parser = SequenceParser()
self.flow_planner = FlowPlanner()
self.defect_rep = DefectReporter()
self.flow_publish = flow_publisher if flow_publisher else PlannedFlowPublisher(
)
self.runner = runner_client if runner_client else RunnerClient(
self.runner_url)
self.page_analyzer = page_analysis_client if page_analysis_client else PageAnalysisClient(
)
self.flow_generator = flow_generator_client if flow_generator_client else FlowGeneratorClient(
)
self.flow_executer = flow_executor if flow_executor else FlowExecutor(
self.form_expert, self.page_analyzer, self.mapper,
self.label_extracter, self.observer, self.defect_rep)
def start(self):
""" Starts the agent control loop.
"""
loop_thread = threading.Thread(target=self.loop_start)
loop_thread.start()
def loop_start(self):
""" Runs the main control loop.
Starts by launching the runner, then executes AgentLoop.NUM_ITERATIONS loop iterations.
:return: Returns once execution has finalized. Also returns if the runner is unable to be launched.
"""
launched = self.runner.launch(self.sut_url)
if not launched:
return
for i in range(AgentLoop.NUM_ITERATIONS):
LOGGER.info(f"Starting loop iteration {str(i)}.")
if general_memory['SESSION_STOPPED']:
LOGGER.info(f"Stopping session due to user stop request.")
break
# noinspection PyBroadException
try:
self.loop_iteration()
except Exception:
LOGGER.exception(f"Fatal error during iteration {str(i)}.")
LOGGER.info(f"Stopping session.")
break
self.loop_end()
def loop_end(self):
""" Finalizes the main control loop, closing any open runner session.
"""
self.runner.quit()
def loop_iteration(self):
""" Executes a single control loop iteration. Each loop iteration performs the following steps:
1) Perceive the current environment, drawing observations.
2) For each observation, consult the flow generation service to generate abstract test flows.
3) For each generated abstract flow, attempt to map it to one or more concrete test flows for the SUT.
4) Publish all concrete test flows to a message queue to be consumed by the Coordinator Agent.
5) If there exists a concrete test flow on our work queue (populated by the Coordinator Agent)
that is currently executable based on current SUT context, pop it off the queue and execute it.
6) Otherwise, check if there is an executable test flow that resulted from Step 3. If so, execute it.
7) Otherwise, go into exploration mode.
:return:
"""
"""First, observe the current environment, extracting information necessary to make a decision."""
concrete_state = self.runner.concrete_state()
if concrete_state is False:
raise Exception("Error during session.")
page_analysis = self.page_analyzer.run_analysis(concrete_state)
abstract_state = self.mapper.process(concrete_state)
LOGGER.debug("Arrived at state: {}.".format(abstract_state.hash))
self.label_extracter.extract_labels(abstract_state, page_analysis)
if len(abstract_state.widgets) == 0:
LOGGER.info("No actionable widgets found. Restarting session.")
self.runner.navigate(self.sut_url)
return
"""
We've arrived to a new actionable state. Run through the following algorithm:
1) Perceive the environment. For all actionable widgets, list observations.
2) Query the flow generator for possible test flows given the observations.
3) For each flow:
a) Execute the flow.
b) Report possible defects.
"""
observations = self.observer.perceive(abstract_state, page_analysis)
"""
Observations are constructed via the ML-based element classifiers,
and are in natural language form (e.g. Observe TextBox FirstName)
"""
test_flow_queue = []
for observation in observations:
LOGGER.debug("Perceived: {}".format(str(observation)))
"""Generated test flows are also in natural language form."""
generated_flow = self.flow_generator.generate_flow(
str(observation))
if generated_flow is not None and generated_flow is not False:
LOGGER.debug("Generated flow: {}".format(generated_flow))
"""Parse the generated flow, constructing an AST leveraging our grammar."""
parsed_flow = self.seq_parser.parse(generated_flow)
if parsed_flow:
"""Leverage the flow planner to convert the abstract flow into concrete flows."""
planned_flows = self.flow_planner.plan(
abstract_state, page_analysis, parsed_flow)
if planned_flows is not False:
"""Extend the internal concrete test flow queue with any planned flows."""
test_flow_queue.extend(planned_flows)
"""Publish all planned flows to the Coordinator Agent."""
if len(test_flow_queue) > 0:
for planned_flow in test_flow_queue:
self.flow_publish.publish(planned_flow)
""" Consume a concrete test flows that has been put on our
Celery-based worker queue by the Coordinator Agent. """
memory_lock.acquire()
if abstract_state.hash in celery_memory:
test_execution_queue = celery_memory[abstract_state.hash]
if len(test_execution_queue) > 0:
planned_flow_to_execute = test_execution_queue.pop(0)
planned_flow_to_execute.calculate_hash()
LOGGER.info(
f'De-queued abstract test off from WORKER QUEUE: {planned_flow_to_execute.hash}'
)
memory_lock.release()
self.flow_executer.execute(abstract_state, self.runner,
planned_flow_to_execute)
"""End the iteration if we've executed a concrete test flow."""
return
else:
memory_lock.release()
else:
memory_lock.release()
"""If we're here, we didn't execute a flow. Are there any on our own internal queue?"""
if len(test_flow_queue) > 0:
LOGGER.debug(
"No abstract tests on WORKER QUEUE. Executing first available abstract test."
)
is_ok = self.flow_executer.execute(abstract_state, self.runner,
random.choice(test_flow_queue))
if not is_ok:
raise Exception("Unable to execute flow step.")
"""End the iteration if we've executed a concrete test flow."""
return
"""If we're here, we still have not executed a flow. Go into EXPLORATION mode."""
if self.memory.in_memory(abstract_state.hash):
chosen_widget = self.memory.choose_widget(abstract_state)
else:
chosen_widget = self.memory.choose_randomly(abstract_state)
self.memory.update_memory(abstract_state, chosen_widget)
if len(chosen_widget["actions"]) == 0:
LOGGER.debug("No actions found on widget: {}. Skipping.".format(
chosen_widget["key"]))
return
action = random.choice(chosen_widget["actions"])
value = None
if action == 'set':
if chosen_widget['label'] is None:
LOGGER.warning(
f"Attempting to set widget ${chosen_widget['key']}, but no widget label found."
)
return
value = self.form_expert.get_concrete_value(chosen_widget['label'])
LOGGER.info("Performing action {} on widget {}.".format(
action, chosen_widget["key"]))
ok = self.runner.perform_action(chosen_widget["selector"], action,
value)
if not ok:
LOGGER.info("Unable to perform action.")
def test_unable_to_perform_runner_click_action():
# Arrange.
form_expert = Mock()
page_analyzer = Mock()
state_abstracter = StateAbstracter()
label_extracter = LabelExtraction()
observer = StateObserver()
defect_rep = Mock()
flow_executor = FlowExecutor(form_expert, page_analyzer, state_abstracter,
label_extracter, observer, defect_rep)
flow_executor.form_fill_strategy = Mock()
to_parse = "OBSERVE TEXTBOX EMAIL TRY VALID EMAIL CLICK COMMIT NOTOBSERVE ERRORMESSAGE"
parser = SequenceParser()
abstract_flow = parser.parse(to_parse)
flow_planner = FlowPlanner()
page_analysis = {
'analysis': {
'labelCandidates': ['Label_Email'],
'COMMIT': ['C1'],
'ERRORMESSAGE': ['E1'],
'errorMessages': ['E1']
}
}
page_analyzer.run_analysis.return_value = page_analysis
abstract_state = ActionableState()
widget_email_label = {
'key': 'Label_Email',
'label': 'EmailLabel',
'actions': [],
'selector': '#lblEmail',
'properties': {
'tagName': 'LABEL',
'text': 'E-mail',
'x': 10,
'y': 10
}
}
widget_email = {
'key': 'EMAIL',
'label': 'Email',
'actions': ['set'],
'selector': '#email',
'properties': {
'tagName': 'INPUT',
'x': 20,
'y': 20
}
}
widget_save = {
'key': 'C1',
'label': 'Save',
'actions': ['click'],
'selector': '#save',
'properties': {
'tagName': 'BUTTON',
'x': 40,
'y': 40
}
}
widget_error = {
'key': 'E1',
'label': 'Error',
'properties': {
'tagName': 'LABEL',
'x': 60,
'y': 60
}
}
abstract_state.add_widget(widget_email_label)
abstract_state.add_widget(widget_email)
abstract_state.add_widget(widget_save)
abstract_state.add_widget(widget_error)
target_concrete_state = {
'widgets': {
'Label_Email': widget_email_label,
'EMAIL': widget_email,
'C1': widget_save,
'E1': widget_error
}
}
runner_mock = Mock()
runner_mock.concrete_state.return_value = target_concrete_state
def click_side_effect(*args, **kwargs):
if args[1] == 'click':
return False
return True
runner_mock.perform_action.side_effect = click_side_effect
planned = flow_planner.plan(abstract_state, page_analysis, abstract_flow)
planned_flow = planned[0]
# Act.
resp = flow_executor.execute(abstract_state, runner_mock, planned_flow)
# Assert.
assert resp is False
