def test_select_dynamic_dom_constraints_feature_switch(self):
success_traces = 'Concolic::ExecutionTree::EndSuccess'
updates = 'Concolic::Solver::DomConstraintsUpdatedDynamically'
ignored = 'Concolic::Solver::SelectDynamicDomConstraintsIgnored'
# N.B. Neither statistic is very intuitive here. the number ignored is often much more than the number of updates. The number of updates can be more than the number of updates which would have made any difference.
report = execute_artemis(
'select_link_value_and_index_feature_switch',
'%sfeature-switches/select-dynamic-form-updates.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(success_traces, 0))
self.assertEqual(5, report.get(updates, 0))
self.assertEqual(0, report.get(ignored, 0))
report = execute_artemis(
'select_link_value_and_index_feature_switch',
'%sfeature-switches/select-dynamic-form-updates.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='select-restriction-dynamic',
verbose=False)
self.assertEqual(0, report.get(success_traces, 0))
self.assertEqual(9, report.get(ignored, 0))
def generate_reports(name, path, exclude):
url = WEBSERVER_URL + "/" + path
print("Starting testing: %s" % name)
print("Running events")
events_r = run_lambda(lambda: execute_artemis(name, url,
iterations=100,
strategy_form_input='random',
strategy_priority='constant',
exclude=exclude))
print("Running const")
const_r = run_lambda(lambda: execute_artemis(name, url,
iterations=100,
strategy_form_input='javascript-constants',
strategy_priority='constant',
exclude=exclude))
print("Running cov")
cov_r = run_lambda(lambda: execute_artemis(name, url,
iterations=100,
strategy_form_input='javascript-constants',
strategy_priority='coverage',
exclude=exclude))
print("Running all")
all_r = run_lambda(lambda: execute_artemis(name, url,
iterations=100,
strategy_form_input='javascript-constants',
strategy_priority='all',
exclude=exclude))
return {
"events": events_r,
"const": const_r,
"cov": cov_r,
"all": all_r}
def test_select_indirect_option_access_feature_switch(self):
key = 'Concolic::Interpreter::IndirectOptionIndexAccess'
report = execute_artemis(
'select_indirect_option_access_feature_switch',
'%sfeature-switches/select-indirect-option-access.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(key, 0))
report = execute_artemis(
'select_indirect_option_access_feature_switch',
'%sfeature-switches/select-indirect-option-access.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='select-indirection-option-index',
verbose=False)
self.assertEqual(0, report.get(key, 0))
def test_select_dynamic_dom_constraints_feature_switch(self):
success_traces = 'Concolic::ExecutionTree::EndSuccess'
updates = 'Concolic::Solver::DomConstraintsUpdatedDynamically'
ignored = 'Concolic::Solver::SelectDynamicDomConstraintsIgnored'
# N.B. Neither statistic is very intuitive here. the number ignored is often much more than the number of updates. The number of updates can be more than the number of updates which would have made any difference.
report = execute_artemis('select_link_value_and_index_feature_switch',
'%sfeature-switches/select-dynamic-form-updates.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(success_traces, 0));
self.assertEqual(5, report.get(updates, 0));
self.assertEqual(0, report.get(ignored, 0));
report = execute_artemis('select_link_value_and_index_feature_switch',
'%sfeature-switches/select-dynamic-form-updates.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='select-restriction-dynamic',
verbose=False)
self.assertEqual(0, report.get(success_traces, 0));
self.assertEqual(9, report.get(ignored, 0));
def test_event_sequence_inject_in_sync_feature_switch(self):
success_traces = 'Concolic::ExecutionTree::EndSuccess'
in_sync = 'Concolic::EventSequence::ChangeHandlersTriggeredInSync'
after_inject = 'Concolic::EventSequence::ChangeHandlersTriggeredAfterInjection'
report = execute_artemis('event_sequence_inject_in_sync_feature_switch',
'%sfeature-switches/select-dynamic-form-updates.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(success_traces, 0));
self.assertEqual(12, report.get(in_sync, 0));
self.assertEqual(0, report.get(after_inject, 0));
report = execute_artemis('event_sequence_inject_in_sync_feature_switch',
'%sfeature-switches/select-dynamic-form-updates.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='event-sequence-sync-injections',
verbose=False)
self.assertEqual(0, report.get(success_traces, 0));
self.assertEqual(0, report.get(in_sync, 0));
self.assertEqual(8, report.get(after_inject, 0)); # Not 12 again because there are less traces executed!
def test_event_sequence_inject_in_sync_feature_switch(self):
success_traces = 'Concolic::ExecutionTree::EndSuccess'
in_sync = 'Concolic::EventSequence::ChangeHandlersTriggeredInSync'
after_inject = 'Concolic::EventSequence::ChangeHandlersTriggeredAfterInjection'
report = execute_artemis(
'event_sequence_inject_in_sync_feature_switch',
'%sfeature-switches/select-dynamic-form-updates.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(success_traces, 0))
self.assertEqual(12, report.get(in_sync, 0))
self.assertEqual(0, report.get(after_inject, 0))
report = execute_artemis(
'event_sequence_inject_in_sync_feature_switch',
'%sfeature-switches/select-dynamic-form-updates.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='event-sequence-sync-injections',
verbose=False)
self.assertEqual(0, report.get(success_traces, 0))
self.assertEqual(0, report.get(in_sync, 0))
self.assertEqual(8, report.get(after_inject, 0))
def test_concrete_value_property_access_feature_switch(self):
ignored_access = 'Concolic::Interpreter::ConcreteValuePropertyAccessIgnored'
traces_explored = 'Concolic::ExecutionTree::DistinctTracesExplored'
report = execute_artemis('concrete_value_property_access_feature_switch',
'%sfeature-switches/hidden-field-concrete-access.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(1, report.get(ignored_access, 0));
self.assertEqual(1, report.get(traces_explored, 0));
report = execute_artemis('concrete_value_property_access_feature_switch',
'%sfeature-switches/hidden-field-concrete-access.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='concrete-value-property',
verbose=False)
self.assertEqual(0, report.get(ignored_access, 0));
self.assertEqual(2, report.get(traces_explored, 0));
def test_cvc4_feature_switch(self):
key = 'Concolic::Solver::SuccessfulCoercionOptimisations'
report = execute_artemis('test_cvc4_feature_switch',
'%sfeature-switches/cvc4-coercion-opt.html' %
FIXTURE_ROOT,
iterations=0,
debug_concolic=' ',
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(key, 0))
report = execute_artemis('test_cvc4_feature_switch',
'%sfeature-switches/cvc4-coercion-opt.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='cvc4-coercion-opt',
verbose=False)
self.assertEqual(0, report.get(key, 0))
def test_concrete_value_property_access_feature_switch(self):
ignored_access = 'Concolic::Interpreter::ConcreteValuePropertyAccessIgnored'
traces_explored = 'Concolic::ExecutionTree::DistinctTracesExplored'
report = execute_artemis(
'concrete_value_property_access_feature_switch',
'%sfeature-switches/hidden-field-concrete-access.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(1, report.get(ignored_access, 0))
self.assertEqual(1, report.get(traces_explored, 0))
report = execute_artemis(
'concrete_value_property_access_feature_switch',
'%sfeature-switches/hidden-field-concrete-access.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='concrete-value-property',
verbose=False)
self.assertEqual(0, report.get(ignored_access, 0))
self.assertEqual(2, report.get(traces_explored, 0))
def test_detect_jquery_eventhandler(self):
report = execute_artemis('test-jquery-live', '%s/jquery-live/index.html' % FIXTURE_ROOT,
strategy_target_selection='jquery',
iterations=2)
self.assertEqual(1, report.get('TargetGeneration::jQuery::eventsIdentified', 0))
report = execute_artemis('test-jquery-live', '%s/jquery-live/index.html' % FIXTURE_ROOT,
strategy_target_selection='legacy',
iterations=2)
self.assertEqual(0, report.get('TargetGeneration::jQuery::eventsIdentified', 0))
def test_readwrite(self):
report = execute_artemis('strategy-priority-readwrite',
'%s/strategies/priority/readwrite.html' % FIXTURE_ROOT,
iterations=4,
strategy_priority='constant')
self.assertEqual(6, report.get('WebKit::coverage::covered-unique', 0));
report = execute_artemis('strategy-priority-readwrite',
'%s/strategies/priority/readwrite.html' % FIXTURE_ROOT,
iterations=4,
strategy_priority='readwrite')
self.assertEqual(7, report.get('WebKit::coverage::covered-unique', 0));
def test_coverage(self):
report = execute_artemis('strategy-priority-coverage',
'%s/strategies/priority/coverage.html' % FIXTURE_ROOT,
iterations=5,
input_strategy_same_length=0,
strategy_priority='constant')
self.assertEqual(8, report.get('WebKit::coverage::covered-unique', 0));
report = execute_artemis('strategy-priority-coverage',
'%s/strategies/priority/coverage.html' % FIXTURE_ROOT,
iterations=5,
strategy_priority='coverage')
self.assertEqual(19, report.get('WebKit::coverage::covered-unique', 0));
def test_detect_jquery_eventhandler(self):
report = execute_artemis('test-jquery-live',
'%s/jquery-live/index.html' % FIXTURE_ROOT,
strategy_target_selection='jquery',
iterations=2)
self.assertEqual(
1, report.get('TargetGeneration::jQuery::eventsIdentified', 0))
report = execute_artemis('test-jquery-live',
'%s/jquery-live/index.html' % FIXTURE_ROOT,
strategy_target_selection='legacy',
iterations=2)
self.assertEqual(
0, report.get('TargetGeneration::jQuery::eventsIdentified', 0))
def test_detect_symbolic_dom_warning(self):
report = execute_artemis('test-symbolic-dom-warning',
'%s/instrumentation/symbolic-dom-warning.html' % FIXTURE_ROOT,
iterations=2)
self.assertEqual(1, report.get('Concolic::SymbolicAPIUsageWarning::JSHTMLElement.dir', 0))
self.assertEqual(1, report.get('Concolic::SymbolicAPIUsageWarning::JSNode.jsNodePrototypeFunctionNormalize', 0))
def test_jsreadwrite(self):
report = execute_artemis(
'instrumentation-jsreadwrite',
'%s/instrumentation/jsreadwrite.html' % WEBSERVER_URL)
self.assertEqual(3, report.get('WebKit::readproperties', 0))
self.assertEqual(5, report.get('WebKit::writtenproperties', 0))
def test_readwrite(self):
report = execute_artemis('strategy-priority-readwrite',
'%s/strategies/priority/readwrite.html' %
WEBSERVER_URL,
iterations=4,
strategy_priority='constant')
self.assertEqual(6, report.get('WebKit::coverage::covered-unique', 0))
report = execute_artemis('strategy-priority-readwrite',
'%s/strategies/priority/readwrite.html' %
WEBSERVER_URL,
iterations=4,
strategy_priority='readwrite')
self.assertEqual(7, report.get('WebKit::coverage::covered-unique', 0))
def _artemis_runner_no_injections(name, path):
return execute_artemis(name,
path,
iterations=2,
debug_concolic=' ',
verbose=False,
event_delegation_testing=True)
def test_jsconstants(self):
report = execute_artemis('instrumentation-jsconstants',
'%s/instrumentation/jsconstants.html' %
FIXTURE_ROOT,
strategy_form_input='javascript-constants')
self.assertEqual(2, report.get('WebKit::jsconstants', 0))
def all_configuration_report(uuid, url, exclude):
report = execute_artemis(uuid, url,
iterations=100,
strategy_form_input='javascript-constants',
strategy_priority='all',
exclude=exclude)
return report
def events_configuration_report(uuid, url, exclude):
report = execute_artemis(uuid, url,
iterations=100,
strategy_form_input='random',
strategy_priority='constant',
exclude=exclude)
return report
def test_set_interval(self):
report = execute_artemis('timer-set-interval', '%s/timers/timer.html' % FIXTURE_ROOT,
input_strategy_same_length=0,
iterations=2)
self.assertEqual(2, report.get('timers::registered', 0))
self.assertEqual(2, report.get('InputGenerator::added-configurations', 0))
self.assertEqual(1, report.get('timers::fired', 0))
def _artemis_runner_full(name, path):
return execute_artemis(name,
path,
iterations=10,
debug_concolic=' ',
strategy_target_selection='concolic',
verbose=False,
event_delegation_testing=True)
def _artemis_runner(name, path):
return execute_artemis(name, path,
iterations=0,
major_mode='concolic-reordering',
concolic_button='id("artform-button")',
concolic_tree_output='final-overview',
event_delegation_testing=False,
verbose=True)
def test_coverage(self):
report = execute_artemis('strategy-priority-coverage',
'%s/strategies/priority/coverage.html' %
WEBSERVER_URL,
iterations=5,
input_strategy_same_length=0,
strategy_priority='constant')
self.assertEqual(8, report.get('WebKit::coverage::covered-unique', 0))
report = execute_artemis('strategy-priority-coverage',
'%s/strategies/priority/coverage.html' %
WEBSERVER_URL,
iterations=5,
strategy_priority='coverage')
self.assertEqual(19, report.get('WebKit::coverage::covered-unique', 0))
def test_form_input_constant(self):
report = execute_artemis('strategy-input-form-constant',
'%s/strategies/inputgeneration/form-input-constant.html' % FIXTURE_ROOT,
strategy_form_input='javascript-constants',
iterations=3)
self.assertEqual(4, report.get('WebKit::coverage::covered-unique', 0));
def test_detect_warning_flag(self):
report = execute_artemis('test-jquery-live',
'%s/instrumentation/dom-warning.html' %
FIXTURE_ROOT,
iterations=2)
self.assertEqual(
1, report.get('DOM::APIUsageWarning::JSNode.parentNode', 0))
def test_basic_sync_call_init(self):
"""
Detect possible ajax callback, but do not call it right now
"""
report = execute_artemis('ajax-basic-sync-call-init', '%s/ajax/index.html' % FIXTURE_ROOT,
iterations=1)
self.assertEqual(1, report.get('InputGenerator::added-configurations', 0));
self.assertEqual(0, report.get("ajax::fired", 0));
self.assertEqual(0, report.get('WebKit::alerts', 0));
def test_form_input_constant(self):
report = execute_artemis(
'strategy-input-form-constant',
'%s/strategies/inputgeneration/form-input-constant.html' %
WEBSERVER_URL,
strategy_form_input='javascript-constants',
iterations=2)
self.assertEqual(4, report.get('WebKit::coverage::covered-unique', 0))
def test_set_interval(self):
report = execute_artemis('timer-set-interval',
'%s/timers/timer.html' % WEBSERVER_URL,
input_strategy_same_length=0,
iterations=2)
self.assertEqual(2, report.get('timers::registered', 0))
self.assertEqual(2,
report.get('InputGenerator::added-configurations', 0))
self.assertEqual(1, report.get('timers::fired', 0))
def test_basic_sync_call(self):
"""
Detect ajax call and call it in iteration 2
"""
report = execute_artemis('ajax-basic-sync-call', '%s/ajax/index.html' % FIXTURE_ROOT,
input_strategy_same_length=0,
iterations=2)
self.assertEqual(2, report.get('InputGenerator::added-configurations', 0));
self.assertEqual(1, report.get("ajax::fired", 0));
self.assertEqual(2, report.get('WebKit::alerts', 0));
def _artemis_runner(name, path):
return execute_artemis(
name,
path,
iterations=0,
debug_concolic=" ",
major_mode="concolic",
concolic_event_sequences="simple",
# concolic_search_procedure='dfs-testing',
verbose=True,
)
def test(self):
if page:
newFilename = setupTempFile(WEBSERVER_ROOT, filename)
else:
newFilename = setUpTempFileFromTemplate(WEBSERVER_ROOT, filename)
report = execute_artemis(name, "{0}/{1}".format(WEBSERVER_URL, newFilename), iterations=5)
if len(report['pathCondition']) > 0:
pc = report['pathCondition'][-1]
else:
pc = ""
self.assertEqual(path_condition.replace(" ", ""), pc.replace(" ", ""))
def _run_test(test_filename, dryrun=False):
name = test_filename.replace('.', '_')
report = execute_artemis(name, "%s/%s" % (WEBSERVER_URL, test_filename),
iterations=2,
major_mode='concolic',
dryrun=dryrun)
if dryrun:
# only print the command, exit
return
def test_basic_sync_call_init(self):
"""
Detect possible ajax callback, but do not call it right now
"""
report = execute_artemis('ajax-basic-sync-call-init',
'%s/ajax/index.html' % WEBSERVER_URL,
iterations=1)
self.assertEqual(1,
report.get('InputGenerator::added-configurations', 0))
self.assertEqual(0, report.get("ajax::fired", 0))
self.assertEqual(0, report.get('WebKit::alerts', 0))
def test_select_link_value_and_index_feature_switch(self):
key = 'Concolic::Solver::SelectConstraintsWithLinkedValueAndIndex'
report = execute_artemis('select_link_value_and_index_feature_switch',
'%sfeature-switches/select-by-index-and-value.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(1, report.get(key, 0));
report = execute_artemis('select_link_value_and_index_feature_switch',
'%sfeature-switches/select-by-index-and-value.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='select-link-value-index',
verbose=False)
self.assertEqual(0, report.get(key, 0));
def test_radio_dom_constraints_feature_switch(self):
key = 'Concolic::Solver::RadioDomConstraintsWritten'
report = execute_artemis('radio_dom_constraints_feature_switch',
'%sfeature-switches/radio-button.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(1, report.get(key, 0));
report = execute_artemis('radio_dom_constraints_feature_switch',
'%sfeature-switches/radio-button.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='radio-restriction',
verbose=False)
self.assertEqual(0, report.get(key, 0));
def test_symbolic_only_after_injection_feature_switch(self):
key = 'Concolic::Interpreter::ConcreteInputValueAccess'
report = execute_artemis('symbolic_only_after_injection_feature_switch',
'%sfeature-switches/select-with-early-usage.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(key, 0));
report = execute_artemis('symbolic_only_after_injection_feature_switch',
'%sfeature-switches/select-with-early-usage.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='symbolic-after-injection',
verbose=False)
self.assertEqual(0, report.get(key, 0));
def test_basic_sync_call(self):
"""
Detect ajax call and call it in iteration 2
"""
report = execute_artemis('ajax-basic-sync-call',
'%s/ajax/index.html' % WEBSERVER_URL,
input_strategy_same_length=0,
iterations=2)
self.assertEqual(2,
report.get('InputGenerator::added-configurations', 0))
self.assertEqual(1, report.get("ajax::fired", 0))
self.assertEqual(1, report.get('WebKit::alerts', 0))
def test_symbolic_checked_property_access_feature_switch(self):
key = 'Concolic::Interpreter::SymbolicCheckedPropertyAccess'
report = execute_artemis('symbolic_select_property_access_feature_switch',
'%sfeature-switches/radio-button.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(key, 0));
report = execute_artemis('symbolic_select_property_access_feature_switch',
'%sfeature-switches/radio-button.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='radio-checkbox-symbolic',
verbose=False)
self.assertEqual(0, report.get(key, 0));
def test_select_indirect_option_access_feature_switch(self):
key = 'Concolic::Interpreter::IndirectOptionIndexAccess'
report = execute_artemis('select_indirect_option_access_feature_switch',
'%sfeature-switches/select-indirect-option-access.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(key, 0));
report = execute_artemis('select_indirect_option_access_feature_switch',
'%sfeature-switches/select-indirect-option-access.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='select-indirection-option-index',
verbose=False)
self.assertEqual(0, report.get(key, 0));
def test_cvc4_feature_switch(self):
key = 'Concolic::Solver::SuccessfulCoercionOptimisations'
report = execute_artemis('test_cvc4_feature_switch',
'%sfeature-switches/cvc4-coercion-opt.html' % FIXTURE_ROOT,
iterations=0,
debug_concolic=' ',
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(key, 0));
report = execute_artemis('test_cvc4_feature_switch',
'%sfeature-switches/cvc4-coercion-opt.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='cvc4-coercion-opt',
verbose=False)
self.assertEqual(0, report.get(key, 0));
def test_select_link_value_and_index_feature_switch(self):
key = 'Concolic::Solver::SelectConstraintsWithLinkedValueAndIndex'
report = execute_artemis(
'select_link_value_and_index_feature_switch',
'%sfeature-switches/select-by-index-and-value.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(1, report.get(key, 0))
report = execute_artemis(
'select_link_value_and_index_feature_switch',
'%sfeature-switches/select-by-index-and-value.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='select-link-value-index',
verbose=False)
self.assertEqual(0, report.get(key, 0))
def _artemis_runner(name,
path,
send_iteration_count=False,
concolic_event_sequences='simple'):
#print "artemis runner", concolic_event_sequences
return execute_artemis(
name,
path,
iterations=0,
debug_concolic=' ',
major_mode='concolic',
concolic_event_sequences=concolic_event_sequences,
#concolic_search_procedure='dfs-testing',
verbose=True,
send_iteration_count=send_iteration_count)
def test_radio_dom_constraints_feature_switch(self):
key = 'Concolic::Solver::RadioDomConstraintsWritten'
report = execute_artemis('radio_dom_constraints_feature_switch',
'%sfeature-switches/radio-button.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(1, report.get(key, 0))
report = execute_artemis('radio_dom_constraints_feature_switch',
'%sfeature-switches/radio-button.html' %
FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='radio-restriction',
verbose=False)
self.assertEqual(0, report.get(key, 0))
def test(self):
if page:
newFilename = setupTempFile(WEBSERVER_ROOT, filename)
else:
newFilename = setUpTempFileFromTemplate(WEBSERVER_ROOT, filename)
report = execute_artemis(name, "{0}/{1}".format(WEBSERVER_ROOT, newFilename), verbose=True, iterations=2,
string_fields=["#selectinput=volvo", "#testinput=1", "#testinput1=1", "#testinput2=1"])
if len(report['pathCondition']) > 0:
pc = report['pathCondition'][-1]
else:
pc = ""
self.assertEqual(path_condition.replace(" ", ""), pc.replace(" ", ""))
def test_symbolic_checked_property_access_feature_switch(self):
key = 'Concolic::Interpreter::SymbolicCheckedPropertyAccess'
report = execute_artemis(
'symbolic_select_property_access_feature_switch',
'%sfeature-switches/radio-button.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(key, 0))
report = execute_artemis(
'symbolic_select_property_access_feature_switch',
'%sfeature-switches/radio-button.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='radio-checkbox-symbolic',
verbose=False)
self.assertEqual(0, report.get(key, 0))
def test_detect_symbolic_dom_warning(self):
report = execute_artemis(
'test-symbolic-dom-warning',
'%s/instrumentation/symbolic-dom-warning.html' % FIXTURE_ROOT,
iterations=2)
self.assertEqual(
1,
report.get('Concolic::SymbolicAPIUsageWarning::JSHTMLElement.dir',
0))
self.assertEqual(
1,
report.get(
'Concolic::SymbolicAPIUsageWarning::JSNode.jsNodePrototypeFunctionNormalize',
0))
def test_symbolic_only_after_injection_feature_switch(self):
key = 'Concolic::Interpreter::ConcreteInputValueAccess'
report = execute_artemis(
'symbolic_only_after_injection_feature_switch',
'%sfeature-switches/select-with-early-usage.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
verbose=False)
self.assertEqual(2, report.get(key, 0))
report = execute_artemis(
'symbolic_only_after_injection_feature_switch',
'%sfeature-switches/select-with-early-usage.html' % FIXTURE_ROOT,
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
concolic_disable_features='symbolic-after-injection',
verbose=False)
self.assertEqual(0, report.get(key, 0))
def test(self):
report = execute_artemis(name, "%s%s" % (FIXTURE_ROOT, filename),
iterations=0,
major_mode='concolic',
concolic_event_sequences='simple',
#concolic_search_procedure='dfs-testing',
verbose=True)
assert test_dict or internal_test, "No tests to execute"
tested_unsat = False
tested_not_written = False
tested_not_solved = False
tested_no_failed_injections = False
if internal_test:
for op, tMap in internal_test.iteritems():
for s, v in tMap.iteritems():
tested_not_written = tested_not_written or s == "Concolic::Solver::ConstraintsNotWritten"
tested_unsat = tested_unsat or s == "Concolic::Solver::ConstraintsSolvedAsUNSAT"
tested_not_solved = tested_not_solved or s == "Concolic::Solver::ConstraintsNotSolved"
tested_no_failed_injections = tested_no_failed_injections or s == "Concolic::FailedInjections"
_assert_test_case(self, op, _get_from_report(report, s)['val'], _get_from_report(report, v)['val'])
if test_dict:
for op, tMap in test_dict.iteritems():
for s, v in tMap.iteritems():
tested_not_written = tested_not_written or s == "Concolic::Solver::ConstraintsNotWritten"
tested_unsat = tested_unsat or s == "Concolic::Solver::ConstraintsSolvedAsUNSAT"
tested_not_solved = tested_not_solved or s == "Concolic::Solver::ConstraintsNotSolved"
tested_no_failed_injections = tested_no_failed_injections or s == "Concolic::FailedInjections"
v = to_appropriate_type(s, v)
r_val = _get_from_report(report, s)
_assert_test_case(self, op, r_val['val'], v.replace(" ", "") if r_val['pc'] else v)
assert tested_unsat or not "Concolic::Solver::ConstraintsSolvedAsUNSAT" in report, \
"Constraints solved as UNSAT are errors pr. default."
assert tested_not_written or not "Concolic::Solver::ConstraintsNotWritten" in report, \
"Not written constraints are pr. default an error"
assert tested_not_solved or not "Concolic::Solver::ConstraintsNotSolved" in report, \
"Not solved constraints are a pr. default an error."
assert tested_no_failed_injections or not "Concolic::FailedInjections" in report, \
"Failed injections are an error by default."
def test(self):
if page:
newFilename = setupTempFile(WEBSERVER_ROOT, filename)
else:
newFilename = setUpTempFileFromTemplate(WEBSERVER_ROOT, filename)
report = execute_artemis(name,
"{0}/{1}".format(WEBSERVER_ROOT, newFilename),
verbose=True,
iterations=2,
string_fields=[
"#selectinput=volvo", "#testinput=1",
"#testinput1=1", "#testinput2=1"
])
if len(report['pathCondition']) > 0:
pc = report['pathCondition'][-1]
else:
pc = ""
self.assertEqual(path_condition.replace(" ", ""), pc.replace(" ", ""))
def test_alert(self):
report = execute_artemis(
'instrumentation-alert',
'%s/instrumentation/alert.html' % WEBSERVER_URL)
self.assertEqual(1, report.get('WebKit::alerts', 0))
def test_no_timers(self):
report = execute_artemis('timer-no-timer', '%s/timers/notimer.html' % FIXTURE_ROOT)
self.assertEqual(0, report.get('timers::registered', 0))
def test_codecoverage_external(self):
report = execute_artemis('instrumentation-codecoverage-external', '%s/instrumentation/codecoverage-external.html' % FIXTURE_ROOT)
self.assertEqual(3, report.get('WebKit::coverage::covered', 0));
def test_jsreadwrite(self):
report = execute_artemis('instrumentation-jsreadwrite', '%s/instrumentation/jsreadwrite.html' % FIXTURE_ROOT)
self.assertEqual(3, report.get('WebKit::readproperties', 0));
self.assertEqual(5, report.get('WebKit::writtenproperties', 0));
def test(self):
def _fetch_and_inject(fields, report):
string_fields = []
boolean_fields = []
integer_fields = []
for field_name in fields:
value = report.get("Concolic::Solver::Constraint.SYM_IN_%s" % field_name, None)
if value is not None:
value = str(value).replace('"', '')
string_fields.append("#%s=%s" % (field_name, value))
continue
value = report.get("Concolic::Solver::Constraint.SYM_IN_BOOL_%s" % field_name, None)
if value is not None:
value = str(value)
value = 'true' if value == 'True' else 'false' if value == 'False' else value
boolean_fields.append("#%s=%s" % (field_name, value))
continue
value = report.get("Concolic::Solver::Constraint.SYM_IN_INT_%s" % field_name, None)
if value is not None:
integer_fields.append("#%s=%s" % (field_name, str(value)))
continue
string_fields.append("#%s=%s" % (field_name, 0))
return string_fields, boolean_fields, integer_fields
unsat = 'unsat' in test_filename
unsupported = 'unsupported' in test_filename
fields = ("testinputselectintlz", "testinputselectint", "testinputx", "testinputy", "testinputNameId", "testinputId", "testinputfoo", "testinputbar", "booleaninput", "selectinput", "radio1a", "radio1b", "radio1c", "testinputselect")
# initial execution
report = execute_artemis(test_name, "%s/%s" % (FIXTURE_ROOT, test_filename),
iterations=2,
debug_concolic=' ',
boolean_fields=["#booleaninput=true", "#radio1b=true", "#radio1a=false", "#radio1c=false"],
string_fields=["#testinputselectintlz=00", "#testinputselectint=0", "#testinputx=1", "#testinputy=2", "#testinputNameId=1", "#testinputId=1", "#testinputfoo=foo", "#testinputbar=bar", "#selectinput=Select1", "#testinputselect=volvo"],
verbose=True)
assert report.get('WebKit::alerts', 0) == 1, "Initial execution did not reach a print statement"
if unsat:
assert report.get('Concolic::Solver::ConstraintsSolvedAsUNSAT', 0) == 1, \
"Initial execution did not return as UNSAT"
return
elif unsupported:
assert report.get('Concolic::Solver::ConstraintsSolved', 0) == 0, \
"Initial execution did not return as unsupported"
return
assert report.get('Concolic::Solver::ErrorsReadingSolution', 0) == 0, "Errors reading the solver solution"
assert report.get('Concolic::Solver::ConstraintsSolvedAsUNSAT', 0) == 0, "Initial execution returned as UNSAT"
assert report.get('Concolic::Solver::ConstraintsNotSolved', 0) == 0, \
"Initial execution returned with an error"
assert report.get('Concolic::Solver::ConstraintsSolved', 0) == 1, "Initial execution did not solve a constraint"
string_fields, boolean_fields, integer_fields = _fetch_and_inject(fields, report)
# repeat positive case
report = execute_artemis(test_name, "%s/%s" % (FIXTURE_ROOT, test_filename),
iterations=2,
debug_concolic=' ',
boolean_fields=boolean_fields,
string_fields=string_fields,
integer_fields=integer_fields,
reverse_constraint_solver=True,
verbose=False)
assert report.get('WebKit::alerts', 0) == 1, \
"Execution using inputs from the solver did not reach a print statement... STRING: %s, BOOLEAN: %s, INTEGER: %s" % (string_fields, boolean_fields, integer_fields)
assert report.get('Concolic::Solver::ErrorsReadingSolution', 0) == 0, "Errors reading the solver solution"
assert report.get('Concolic::Solver::ConstraintsSolvedAsUNSAT', 0) == 0, "NEGATED execution returned as UNSAT"
assert report.get('Concolic::Solver::ConstraintsNotSolved', 0) == 0, \
"NEGATED execution returned with an error"
assert report.get('Concolic::Solver::ConstraintsSolved', 0) == 1, \
"NEGATED execution did not solve a constraint"
# negative case
string_fields, boolean_fields, integer_fields = _fetch_and_inject(fields, report)
report = execute_artemis(test_name, "%s/%s" % (FIXTURE_ROOT, test_filename),
iterations=2,
debug_concolic=' ',
boolean_fields=boolean_fields,
string_fields=string_fields,
integer_fields=integer_fields,
verbose=False)
assert report.get('WebKit::alerts', 0) == 0, \
"NEGATED execution REACHED a print statement when it should not using STRING: %s, BOOLEAN: %s, INTEGER: %s" % (string_fields, boolean_fields, integer_fields)
def test_jsconstants(self):
report = execute_artemis('instrumentation-jsconstants', '%s/instrumentation/jsconstants.html' % FIXTURE_ROOT,
strategy_form_input='javascript-constants')
self.assertEqual(2, report.get('WebKit::jsconstants', 0));
