def des_request_param_payload(request_param_payload_json):
""" Deserialize RequestParametersPayload type object.
@param request_param_payload_json: Request parameter from the compiler
@type request_param_payload_json: JSON
@return: List of tuples containing the keys and payloads
@rtype: List[tuple(str, ParamObject)]
"""
KeyPayload = collections.namedtuple("KeyPayload", ['key', 'payload'])
payloads = []
if 'ParameterList' in request_param_payload_json:
param_list_seq = request_param_payload_json['ParameterList']
for param_payload_pair in param_list_seq:
# pair must have size 2
if len(param_payload_pair) != 2:
logger.write_to_main(
'string - param payload pair size mismatch')
return [KeyPayload(None, None)]
key = param_payload_pair[0]
payload = param_payload_pair[1]
payloads.append(KeyPayload(key, payload))
return payloads
return [KeyPayload(None, None)]
def __init__(self, request_schema_json):
""" Initialize and construct the RequestExamples by deserializing the
compiler generated request schema IL.
@param request_schema_json: Compiler generated request schema IL
@type request_schema_json: JSON
@return: None
@rtype: None
"""
# initialization
self._query_examples: set = set() # {QueryList}
self._body_examples: set = set() # {BodySchema}
# process the request schema
try:
self._set_query_params(request_schema_json['queryParameters'])
except Exception as err:
msg = f'Fail deserializing request schema query examples: {err!s}'
logger.write_to_main(msg, print_to_console=True)
raise Exception(msg)
try:
self._set_body_params(request_schema_json['bodyParameters'])
except Exception as err:
msg = f'Fail deserializing request schema body examples: {err!s}'
logger.write_to_main(msg, print_to_console=True)
raise Exception(msg)
if not self._query_examples and not self._body_examples:
raise NoExamplesFound
def des_request_param_payload(request_param_payload_json):
""" Deserialize RequestParametersPayload type object.
@param request_param_payload_json: Request parameter from the compiler
@type request_param_payload_json: JSON
@return: List of tuples containing the keys and payloads
@rtype: List[tuple(str, ParamObject)]
"""
KeyPayload = collections.namedtuple("KeyPayload", ['key', 'payload'])
payloads = []
if 'ParameterList' in request_param_payload_json:
param_list_seq = request_param_payload_json['ParameterList']
for param_payload_pair in param_list_seq:
if not ('name' in param_payload_pair
and 'payload' in param_payload_pair):
logger.write_to_main(
'string - param payload does not contain expected elements'
)
raise Exception(
"Error parsing param payload json. See the main log for more details."
)
key = param_payload_pair['name']
payload = param_payload_pair['payload']
payloads.append(KeyPayload(key, payload))
return payloads
return [KeyPayload(None, None)]
def __init__(self, request_schema_json=None, fuzzing_config=None, param=None):
""" Initialize and construct the BodySchema by deserializing the
compiler generated request schema.
@param request_schema_json: Compiler generated request schema
@type request_schema_json: JSON
@param fuzzing_config: PayloadBodyChecker specific configuration data (can be None)
@type fuzzing_config: Dict
@param param: Body schema as a ParamObject
@type param: ParamObject
@return: None
@rtype: None
"""
self._schema = param
# Used by payload body checker
# Config is used for stateful monitoring while traversing the schema
self._config = FuzzingConfig(fuzzing_config)
if request_schema_json:
try:
self._set_body_schema(request_schema_json['bodyParameters'])
except NoSchemaFound:
raise
except Exception as err:
msg = f'Fail deserializing request schema body parameters: {err!s}'
logger.write_to_main(msg, print_to_console=True)
raise Exception(msg)
self._node_count = self._schema.count_nodes(self._config)
def _handle_exception(type, tag, err):
logger.write_to_main(
f"Exception when rendering request {self.method} {self.endpoint_no_dynamic_objects}.\n"
f"Type: {type}. Tag: {tag}.\n"
f" Exception: {err!s}",
print_to_console=True)
raise InvalidDictionaryException
def call_response_parser(parser, response, request=None):
""" Calls a specified parser on a response
@param parser: The parser function to calls
@type parser: Func
@param response: The response to parse
@type response: HttpResponse
@param request: The request whose parser is being called
@type request: Request (None ok)
@return False if there was a parser exception
@rtype Boolean
"""
from utils.logger import write_to_main
# parse response and set dependent variables (for garbage collector)
try:
if parser:
parser(response.json_body)
# Check request's producers to verify dynamic objects were set
if request:
for producer in request.produces:
if dependencies.get_variable(producer) == 'None':
err_str = f'Failed to parse {producer}; is now set to None.'
write_to_main(err_str)
_RAW_LOGGING(err_str)
except (ResponseParsingException, AttributeError) as error:
_RAW_LOGGING(str(error))
return False
return True
def _raise_dict_err(type, tag):
logger.write_to_main(
f"Error for request {self.method} {self.endpoint_no_dynamic_objects}.\n"
f"{type} exception: {tag} not found.\n"
"Make sure you are using the dictionary created during compilation.",
print_to_console=True)
raise InvalidDictionaryException
def add_producers(req, req_collection, req_list, dfs_stack):
# cycle detection
# Note: the next line is linear in dfs_stack but dfs_stack is usually very small (<10)
# Note: dfs_stack is used as, and could be replaced by, a set here
if req in dfs_stack:
report_dependency_cycle(req, dfs_stack)
return []
# else append req to dfs_stack
dfs_stack.append(req)
# Find all other requests producing objects consumed by req
for consumer_var_name in sorted(req.consumes):
# Find the producer of this object
producer_found = False
for producer_req in req_collection:
if consumer_var_name in producer_req.produces:
# Producer is found
if producer_found == True:
logger.write_to_main(
f"\nError in input grammar: {consumer_var_name} has more than one producer!\n",
True)
return []
producer_found = True
# If this producer is not already in req_list, add its own producers recursively...
# Note: the next line is linear in req_list but req_list is usually very small (<10)
if producer_req not in req_list:
req_list = add_producers(producer_req, req_collection,
req_list, dfs_stack)
# if an error occurred in the sub-computation below, abort and pop-up
if req_list == []:
return []
if producer_found == False:
logger.write_to_main(
f"\nError in input grammar: {consumer_var_name} has no producer!\n",
True)
return []
# Since all producers of req (if any) have been processed, add req at the end of req_list
req_list.append(req)
# continue the recursion by popping up
dfs_stack.pop()
return req_list
def report_dependency_cycle(request, req_list):
logger.write_to_main(
"\nError in input grammar: a request is in a circular dependency!"
f"\nRequest: {request.method} {request.endpoint}"
"\n\tThe circular dependency is in the following request sequence:",
True)
for req in req_list:
logger.write_to_main(f"\n\tRequest: {req.method} {req.endpoint}", True)
for consumer_var_name in req.consumes:
logger.write_to_main(f"\t\tConsumes {consumer_var_name}", True)
for producer_var_name in req.produces:
logger.write_to_main(f"\t\tProduces {producer_var_name}", True)
def parse_grammar_schema(schema_json):
""" Parses the grammar.json file for examples and body schemas and sets the
examples for each matching request in the RequestCollection
@param schema_json: The json schema to parse for examples
@type schema_json: Json
@return: False if there was an exception while parsing the examples
@rtype : Bool
"""
try:
# Process request schema by looping through each request in grammar.json
for request_schema_json in schema_json['Requests']:
method = request_schema_json['method'].upper()
endpoint = request_schema_json['id']['endpoint']
# Parse json using the RequestExamples class to create a collection of
# examples for each request.
try:
request_examples = RequestExamples(request_schema_json)
except NoExamplesFound:
# No examples exist for this request
request_examples = None
try:
body_schema = BodySchema(request_schema_json)
except NoSchemaFound:
# No body schema exists for this request
body_schema = None
if request_examples or body_schema:
_set_schemas(request_examples, body_schema, method, endpoint)
return True
except ValueError as err:
logger.write_to_main(
f"Failed to parse grammar file for examples: {err!s}",
print_to_console=True)
return False
def set_request_stats(self, request_text):
""" Helper to set the request statistics from the text.
Parses the request text and initializes headers, uri, and body
separately.
@return: None
@rtype : None
"""
try:
split_body = request_text.split(messaging.DELIM)
split_headers = split_body[0].split("\r\n")
self.request_uri = split_headers[0].split(" ")[1]
self.request_headers = split_headers[1:]
if len(split_body) > 0 and split_body[1]:
self.request_body = split_body[1]
except:
logger.write_to_main(
f"Error setting request stats for text: {request_text}",
print_to_console=True
)
pass
def generate_sequences(fuzzing_requests, checkers, fuzzing_jobs=1):
""" Implements core restler algorithm.
@param fuzzing_requests: The collection of requests that will be fuzzed
@type fuzzing_requests: FuzzingRequestCollection
@param checkers: The list of checkers to apply
@type checkers: list[Checker]
@param fuzzing_jobs: Optional number of fuzzing jobs for parallel fuzzing.
Default value passed is one (sequential fuzzing).
@type fuzzing_jobs: Int
@return: None
@rtype : None
"""
if not fuzzing_requests.size:
return
logger.create_network_log(logger.LOG_TYPE_TESTING)
fuzzing_mode = Settings().fuzzing_mode
max_len = Settings().max_sequence_length
if fuzzing_mode == 'directed-smoke-test':
return generate_sequences_directed_smoketest(fuzzing_requests,
checkers)
if fuzzing_jobs > 1:
render = render_parallel
global_lock = multiprocessing.Lock()
fuzzing_pool = ThreadPool(fuzzing_jobs)
else:
global_lock = None
fuzzing_pool = None
render = render_sequential
should_stop = False
timeout_reached = False
seq_collection_exhausted = False
num_total_sequences = 0
while not should_stop:
seq_collection = [sequences.Sequence()]
# Only for bfs: If any checkpoint file is available, load state of
# latest generation. Note that it only makes sense to use checkpoints
# for the bfs exploration method, since it is the only systemic and
# exhaustive method.
min_len = 0
if fuzzing_mode == 'bfs':
req_collection = GrammarRequestCollection()
monitor = Monitor()
req_collection, seq_collection, fuzzing_requests, monitor, min_len =\
saver.load(req_collection, seq_collection, fuzzing_requests, monitor)
requests.GlobalRequestCollection.Instance(
)._req_collection = req_collection
fuzzing_monitor.FuzzingMonitor.__instance = monitor
# Repeat external loop only for random walk
if fuzzing_mode != 'random-walk':
should_stop = True
# Initialize fuzzing schedule
fuzzing_schedule = {}
logger.write_to_main(f"Setting fuzzing schemes: {fuzzing_mode}")
for length in range(min_len, max_len):
fuzzing_schedule[length] = fuzzing_mode
# print(" - {}: {}".format(length + 1, fuzzing_schedule[length]))
# print general request-related stats
logger.print_req_collection_stats(
fuzzing_requests,
GrammarRequestCollection().candidate_values_pool)
generation = 0
for length in range(min_len, max_len):
# we can set this without locking, since noone else writes (main
# driver is single-threaded) and every potential worker will just
# read-access this value.
generation = length + 1
fuzzing_mode = fuzzing_schedule[length]
# extend sequences with new request templates
seq_collection = extend(seq_collection, fuzzing_requests,
global_lock)
print(f"{formatting.timestamp()}: Generation: {generation} ")
logger.write_to_main(
f"{formatting.timestamp()}: Generation: {generation} / "
f"Sequences Collection Size: {len(seq_collection)} "
f"(After {fuzzing_schedule[length]} Extend)")
# render templates
try:
seq_collection_exhausted = False
seq_collection = render(seq_collection, fuzzing_pool, checkers,
generation, global_lock)
except TimeOutException:
logger.write_to_main("Timed out...")
timeout_reached = True
seq_collection_exhausted = True
# Increase fuzzing generation after timeout because the code
# that does it would have never been reached. This is done so
# the previous generation's test summary is logged correctly.
Monitor().current_fuzzing_generation += 1
except ExhaustSeqCollectionException:
logger.write_to_main("Exhausted collection...")
seq_collection = []
seq_collection_exhausted = True
logger.write_to_main(
f"{formatting.timestamp()}: Generation: {generation} / "
f"Sequences Collection Size: {len(seq_collection)} "
f"(After {fuzzing_schedule[length]} Render)")
# saving latest state
saver.save(GrammarRequestCollection(), seq_collection,
fuzzing_requests, Monitor(), generation)
# Print stats for iteration of the current generation
logger.print_generation_stats(GrammarRequestCollection(),
Monitor(), global_lock)
num_total_sequences += len(seq_collection)
logger.print_request_rendering_stats(
GrammarRequestCollection().candidate_values_pool,
fuzzing_requests, Monitor(),
Monitor().num_fully_rendered_requests(
fuzzing_requests.all_requests), generation, global_lock)
if timeout_reached or seq_collection_exhausted:
if timeout_reached:
should_stop = True
break
logger.write_to_main("--\n")
if fuzzing_pool is not None:
fuzzing_pool.close()
fuzzing_pool.join()
return num_total_sequences
def generate_sequences_directed_smoketest(fuzzing_requests, checkers):
""" Checks whether each request can be successfully rendered.
For each request:
- Constructs a sequence that satisfies all dependencies by backtracking.
- Renders this sequence.
This allows debugging rendering on a per-request basis
to resolve configuration or spec issues.
"""
def render_request(request, seq):
""" Helper function that attempts to find a valid rendering for the request.
The do-while loop will render each combination of the request until either
a valid rendering is detected or all combinations have been exhausted.
Side effects: request.stats.status_code updated
request.stats.status_text updated
request.stats updated with concrete response and request text
(valid request or last combination)
@return: Tuple containing rendered sequence object, response body, and
failure information enum.
@rtype : Tuple(RenderedSequence, str, FailureInformation)
"""
response_body = None
rendering_information = None
while True:
renderings = seq.render(candidate_values_pool, global_lock)
if renderings.failure_info:
# Even though we will be returning renderings from this function,
# the renderings object that is returned may be from an unrendered
# sequence. We want to save the most recent info.
rendering_information = renderings.failure_info
# Perform this check/save here in case the last call to seq.render
# returns an empty 'renderings' object. An empty renderings object
# will be returned from seq.render if all request combinations are
# exhausted prior to getting a valid status code.
if renderings.final_request_response:
request.stats.status_code = renderings.final_request_response.status_code
request.stats.status_text = renderings.final_request_response.status_text
# Get the last rendered request. The corresponding response should be
# the last received response.
request.stats.sample_request.set_request_stats(
renderings.sequence.sent_request_data_list[-1].
rendered_data)
request.stats.sample_request.set_response_stats(
renderings.final_request_response,
renderings.final_response_datetime)
response_body = renderings.final_request_response.body
apply_checkers(checkers, renderings, global_lock)
# If a valid rendering was found or the combinations have been
# exhausted (empty rendering), exit the loop.
if renderings.valid or renderings.sequence is None:
return renderings, response_body, rendering_information
global_lock = None
candidate_values_pool = GrammarRequestCollection().candidate_values_pool
# print general request-related stats
logger.print_req_collection_stats(
fuzzing_requests,
GrammarRequestCollection().candidate_values_pool)
logger.write_to_main(
f"\n{formatting.timestamp()}: Starting directed-smoke-test\n")
# Sort the request list prior to computing the request sequences,
# so the prefixes are always in the same order for the algorithm
fuzzing_request_list = list(fuzzing_requests._requests)
fuzzing_request_list.sort(key=lambda x: x.method_endpoint_hex_definition)
# sort the requests in fuzzing_requests by depth
sorted_fuzzing_req_list = []
for request in fuzzing_request_list:
req_list = compute_request_goal_seq(request, fuzzing_request_list)
if len(req_list) > 0:
sorted_fuzzing_req_list.append([len(req_list), request, req_list])
# Else an error message was printed and we skip this request
# now sort by length (secondary sort by a hash of the request definition text)
sorted_fuzzing_req_list.sort(
key=lambda x: (x[0], x[1].method_endpoint_hex_definition))
logger.write_to_main(f"{formatting.timestamp()}: Will attempt to render "
f"{len(sorted_fuzzing_req_list)} requests found\n")
# the two following lists are indexed by request number and are of the same size.
# memoize valid rendered sequences for each request and re-use those when going deeper
valid_rendered_sequences_list = []
# memoize the first invalid prefix for each request
first_invalid_prefix_list = []
# try to render all requests starting with the shallow ones
for idx, request_triple in enumerate(sorted_fuzzing_req_list):
req_list_length = request_triple[0]
request = request_triple[1]
req_list = request_triple[2]
valid = False
first_invalid_prefix = -1 # -1 denotes no invalid prefix by default
request.stats.request_order = idx
Found = False
if (req_list_length > 1):
# search for a valid matching prefix we can re-use;
# unless path_regex is used we should always find a match
# because we start with shallow sequences
req_list_prefix = req_list[:-1]
i = 0
while (not Found) and (i < idx):
if sorted_fuzzing_req_list[i][2] == req_list_prefix:
# we found a match
Found = True
logger.write_to_main(
f"Found a matching prefix for request {idx} with previous request {i}"
)
request.stats.matching_prefix[
"id"] = sorted_fuzzing_req_list[i][
1].method_endpoint_hex_definition
else:
# continue searching
i = i + 1
rendering_information = None
response_body = None
if Found:
if valid_rendered_sequences_list[i].is_empty_sequence():
# then the current sequence will also be INVALID.
# propagate the root-cause explaining why the prefix was invalid
first_invalid_prefix = first_invalid_prefix_list[i]
logger.write_to_main(
f"\tbut that prefix was INVALID (root = {first_invalid_prefix})\n"
)
request.stats.matching_prefix["valid"] = 0
# since valid = False by default, nothing else to do here
else:
# re-use the previous VALID prefix
logger.write_to_main("\tand re-using that VALID prefix\n")
request.stats.matching_prefix["valid"] = 1
new_seq = valid_rendered_sequences_list[i]
req_copy = copy.copy(request)
req_copy._current_combination_id = 0
new_seq = new_seq + sequences.Sequence(req_copy)
new_seq.seq_i = 0
renderings, response_body, rendering_information = render_request(
request, new_seq)
valid = renderings.valid
else:
logger.write_to_main(f"Rendering request {idx} from scratch\n")
# render the sequence.
new_seq = sequences.Sequence()
for req in req_list:
req_copy = copy.copy(req)
req_copy._current_combination_id = 0
if new_seq.is_empty_sequence():
new_seq = sequences.Sequence(req_copy)
else:
new_seq = new_seq + sequences.Sequence(req_copy)
new_seq.seq_i = 0
renderings, response_body, rendering_information = render_request(
req, new_seq)
valid = renderings.valid
logger.write_to_main(
f"{formatting.timestamp()}: Request {idx}\n"
f"{formatting.timestamp()}: Endpoint - {request.endpoint_no_dynamic_objects}\n"
f"{formatting.timestamp()}: Hex Def - {request.method_endpoint_hex_definition}\n"
f"{formatting.timestamp()}: Sequence length that satisfies dependencies: {req_list_length}"
)
if valid:
logger.write_to_main(f"{formatting.timestamp()}: Rendering VALID")
request.stats.valid = 1
# remember this valid sequence
valid_rendered_sequences_list.append(new_seq)
first_invalid_prefix_list.append(first_invalid_prefix)
else:
logger.write_to_main(
f"{formatting.timestamp()}: Rendering INVALID")
request.stats.valid = 0
request.stats.error_msg = response_body
# remember RESTler didn't find any valid sequence with an empty request sequence
valid_rendered_sequences_list.append(sequences.Sequence())
if (first_invalid_prefix == -1):
first_invalid_prefix = idx
first_invalid_prefix_list.append(first_invalid_prefix)
if rendering_information:
if rendering_information == FailureInformation.PARSER:
msg = (
"This request received a VALID status code, but the parser failed.\n"
"Because of this, the request was set to INVALID.\n")
elif rendering_information == FailureInformation.RESOURCE_CREATION:
msg = (
"This request received a VALID status code, but the server "
"indicated that there was a failure when creating the resource.\n"
)
elif rendering_information == FailureInformation.SEQUENCE:
msg = (
"This request was never rendered because the sequence failed to re-render.\n"
"Because of this, the request was set to INVALID.\n")
elif rendering_information == FailureInformation.BUG:
msg = "A bug code was received after rendering this request."
else:
msg = "An unknown error occurred when processing this request."
logger.write_to_main(f"{formatting.timestamp()}: {msg}")
request.stats.failure = rendering_information
rendering_information = None
logger.format_rendering_stats_definition(
request,
GrammarRequestCollection().candidate_values_pool)
logger.print_request_rendering_stats(
GrammarRequestCollection().candidate_values_pool, fuzzing_requests,
Monitor(), fuzzing_requests.size_all_requests,
Monitor().current_fuzzing_generation, global_lock)
Monitor().current_fuzzing_generation += 1
return len(valid_rendered_sequences_list)
def compute_request_goal_seq(request, req_collection):
req_list = add_producers(request, req_collection, [], [])
if req_list == []:
logger.write_to_main(
f"\nSkipping request {request.method} {request.endpoint}\n", True)
return req_list
def delete_create_once_resources(destructors, fuzzing_requests):
""" Iterates through each destructor request and sends it to the server
@param destructors: A list of destructor requests to send
@type destructors: list(Request)
@param fuzzing_requests: The global collection of requests to fuzz
@type fuzzing_requests: FuzzingRequestCollection
@return: None
@rtype : None
"""
if not destructors:
return
candidate_values_pool = GrammarRequestCollection().candidate_values_pool
logger.write_to_main("\nRendering for create-once resource destructors:\n")
for destructor in destructors:
status_codes = []
try:
logger.write_to_main(
f"{formatting.timestamp()}: Endpoint - {destructor.endpoint_no_dynamic_objects}"
)
logger.write_to_main(
f"{formatting.timestamp()}: Hex Def - {destructor.method_endpoint_hex_definition}"
)
seq = sequences.Sequence([destructor])
renderings = seq.render(
GrammarRequestCollection().candidate_values_pool,
None,
postprocessing=True)
if not renderings.valid:
logger.write_to_main(
f"{formatting.timestamp()}: Rendering INVALID")
else:
logger.write_to_main(
f"{formatting.timestamp()}: Rendering VALID")
logger.format_rendering_stats_definition(
destructor,
GrammarRequestCollection().candidate_values_pool)
if Settings().in_smoke_test_mode():
destructor.stats.request_order = 'Postprocessing'
destructor.stats.valid = 1
destructor.stats.status_code = renderings.final_request_response.status_code
destructor.stats.status_text = renderings.final_request_response.status_text
destructor.stats.sample_request.set_request_stats(
renderings.sequence.sent_request_data_list[-1].
rendered_data)
destructor.stats.sample_request.set_response_stats(
renderings.final_request_response,
renderings.final_response_datetime)
except Exception as error:
msg = f"Failed to delete create_once resource: {error!s}"
logger.raw_network_logging(msg)
logger.write_to_main(msg, print_to_console=True)
if Settings().in_smoke_test_mode():
destructor.stats.request_order = 'Postprocessing'
destructor.stats.valid = 0
if renderings and renderings.final_request_response:
destructor.stats.status_code = renderings.final_request_response.status_code
destructor.stats.status_text = renderings.final_request_response.status_text
destructor.stats.error_msg = renderings.final_request_response.body
destructor.stats.sample_request.set_request_stats(
renderings.sequence.sent_request_data_list[-1].
rendered_data)
destructor.stats.sample_request.set_response_stats(
renderings.final_request_response,
renderings.final_response_datetime)
pass
Monitor().current_fuzzing_generation += 1
logger.print_request_rendering_stats(candidate_values_pool,
fuzzing_requests, Monitor(),
fuzzing_requests.size_all_requests,
logger.POSTPROCESSING_GENERATION,
None)
def des_param_payload(param_payload_json, tag='', query_param=False):
""" Deserialize ParameterPayload type object.
@param param_payload_json: Body parameter from the compiler
@type param_payload_json: JSON
@param tag: Node tag
@type tag: Str
@param query_param: Set to True if this is a query parameter
@type query_param: Bool
@return: Body parameter schema
@rtype: ParamObject
"""
param = None
if 'InternalNode' in param_payload_json:
internal_node = param_payload_json['InternalNode']
internal_info = internal_node[0]
internal_data = internal_node[1]
name = internal_info['name']
property_type = internal_info['propertyType']
if tag:
next_tag = tag + '_' + name
else:
next_tag = name
# Array --> ParamMember { name : ParamArray }
if property_type == 'Array':
values = []
for data in internal_data:
value = des_param_payload(data, next_tag)
values.append(value)
array = ParamArray(values)
if query_param or not name:
param = array
else:
param = ParamMember(name, array)
array.tag = f'{next_tag}_array'
# Object --> ParamObject { ParamMember, ..., ParamMember }
elif property_type == 'Object':
members = []
for member_json in internal_data:
member = des_param_payload(member_json, tag)
members.append(member)
param = ParamObject(members)
param.tag = f'{next_tag}_object'
# Property --> ParamMember { name : ParamObject }
elif property_type == 'Property':
if len(internal_data) != 1:
logger.write_to_main(f'Internal Property {name} size != 1')
value = des_param_payload(internal_data[0], next_tag)
param = ParamMember(name, value)
# others
else:
logger.write_to_main(f'Unknown internal type {property_type}')
elif 'LeafNode' in param_payload_json:
leaf_node = param_payload_json['LeafNode']
name = leaf_node['name']
payload = leaf_node['payload']
# payload is a dictionary (or member) with size 1
if len(payload) != 1:
logger.write_to_main(f'Unexpected payload format {payload}')
content_type = 'Unknown'
content_value = 'Unknown'
custom = False
fuzzable = False
if 'Fuzzable' in payload:
content_type = payload['Fuzzable'][0]
content_value = payload['Fuzzable'][1]
fuzzable = True
elif 'Constant' in payload:
content_type = payload['Constant'][0]
content_value = payload['Constant'][1]
elif 'DynamicObject' in payload:
content_type = 'Unknown'
content_value = payload['DynamicObject']
elif 'Custom' in payload:
custom = True
content_type = payload['Custom']['payloadType']
content_value = payload['Custom']['payloadValue']
elif 'PayloadParts' in payload:
definition = payload['PayloadParts'][-1]
if 'Custom' in definition:
custom = True
content_type = definition['Custom']['payloadType']
content_value = definition['Custom']['payloadValue']
# create value w.r.t. the type
value = None
if content_type == 'String':
# If query parameter, assign as a value and not a string
# because we don't want to wrap with quotes in the request
if query_param:
value = ParamValue(custom=custom)
else:
value = ParamString(custom)
elif content_type == 'DateTime':
value = ParamString(custom)
elif content_type == 'Int':
value = ParamNumber()
elif content_type == 'Number':
value = ParamNumber()
elif content_type == 'Bool':
value = ParamBoolean()
elif content_type == 'Object':
value = ParamObjectLeaf()
elif content_type == 'UuidSuffix':
value = ParamUuidSuffix()
# Set as unknown for payload body fuzzing purposes.
# This will be fuzzed as a string.
value.set_unknown()
elif 'Enum' in content_type:
# unique case for Enums, as they are defined as
# "fuzzable" types in the schema, but are not fuzzable
# by the same definition as the rest of the fuzzable types.
fuzzable = False
# {
# Enum : [
# type,
# [ value1, value2, value3 ],
# default_value
# ]
# }
enum_definition = content_type['Enum']
if len(enum_definition) == 3:
enum_content_type = enum_definition[0]
contents = enum_definition[1]
value = ParamEnum(contents, enum_content_type)
else:
logger.write_to_main(f'Unexpected enum schema {name}')
else:
value = ParamString(False)
value.set_unknown()
value.set_fuzzable(fuzzable)
value.content = content_value
if tag and name:
value.tag = (tag + '_' + name)
elif tag:
value.tag = tag
else:
value.tag = name
# create the param node
if name:
param = ParamMember(name, value)
else:
# when a LeafNode represent a standard type, e.g.,
# string, the name will be empty
param = value
else:
logger.write_to_main('Neither internal nor leaf property')
if not param:
logger.write_to_main(f'Fail des param payload {param_payload_json}')
return None
return param
def render(self,
candidate_values_pool,
lock,
preprocessing=False,
postprocessing=False):
""" Core routine that performs the rendering of restler sequences. In
principal all requests of a sequence are being constantly rendered with
a specific values combination @param request._current_combination_id
which we know in the past led to a valid rendering and only the last
request of the sequence is being rendered iteratively with all feasible
value combinations. Each time a "valid rendering" is found for the last
request of the sequence (where "valid rendering" is defined according
to "VALID_CODES"), the routine returns a new sequence which has an
end-to-end (i.e., all requests) "valid rendering" and can be added in
the sequences collection in order to be used in the future as a building
block for longer sequences.
@param candidate_values_pool: The pool of values for primitive types.
@type candidate_values_pool: Dict
@param lock: Lock object used for sync of more than one fuzzing jobs.
@type lock: thread.Lock object
@param preprocessing: Set to true if rendering during preprocessing
@type preprocessing: Bool
@return: A RenderedSequence object containing the sequence, the final
request's response, whether or not the final request received
a valid status code, and a FailureInformation enum if there was
a failure or bug detected during rendering.
@rtype : RenderedSequence
"""
# Try rendering all primitive type value combinations for last request
request = self.last_request
# for clarity reasons, don't log requests whose render iterator is over
if request._current_combination_id <\
request.num_combinations(candidate_values_pool):
CUSTOM_LOGGING(self, candidate_values_pool)
self._sent_request_data_list = []
for rendered_data, parser in\
request.render_iter(candidate_values_pool,
skip=request._current_combination_id,
preprocessing=preprocessing):
# Hold the lock (because other workers may be rendering the same
# request) and check whether the current rendering is known from the
# past to lead to invalid status codes. If so, skip the current
# rendering.
if lock is not None:
lock.acquire()
should_skip = Monitor().is_invalid_rendering(request)
if lock is not None:
lock.release()
# Skip the loop and don't forget to increase the counter.
if should_skip:
RAW_LOGGING("Skipping rendering: {}".\
format(request._current_combination_id))
request._current_combination_id += 1
continue
# Clean up internal state
self.status_codes = []
dependencies.reset_tlb()
sequence_failed = False
# Step A: Static template rendering
# Render last known valid combination of primitive type values
# for every request until the last
for i in range(len(self.requests) - 1):
prev_request = self.requests[i]
prev_rendered_data, prev_parser =\
prev_request.render_current(candidate_values_pool,
preprocessing=preprocessing)
# substitute reference placeholders with resolved values
if not Settings().ignore_dependencies:
prev_rendered_data =\
self.resolve_dependencies(prev_rendered_data)
prev_req_async_wait = Settings(
).get_max_async_resource_creation_time(prev_request.request_id)
prev_producer_timing_delay = Settings(
).get_producer_timing_delay(prev_request.request_id)
prev_response = request_utilities.send_request_data(
prev_rendered_data)
prev_response_to_parse, resource_error, async_waited = async_request_utilities.try_async_poll(
prev_rendered_data, prev_response, prev_req_async_wait)
prev_parser_threw_exception = False
# Response may not exist if there was an error sending the request or a timeout
if prev_parser and prev_response_to_parse:
prev_parser_threw_exception = not request_utilities.call_response_parser(
prev_parser, prev_response_to_parse, prev_request)
prev_status_code = prev_response.status_code
# If the async logic waited for the resource, this wait already included the required
# producer timing delay. Here, set the producer timing delay to zero, so this wait is
# skipped both below for this request and during replay
if async_waited:
prev_producer_timing_delay = 0
else:
prev_req_async_wait = 0
self.append_data_to_sent_list(prev_rendered_data, prev_parser,
prev_response,
prev_producer_timing_delay,
prev_req_async_wait)
if not prev_status_code:
logger.write_to_main(
f"Error: Failed to get status code during valid sequence re-rendering.\n"
)
sequence_failed = True
break
if prev_response.has_bug_code():
BugBuckets.Instance().update_bug_buckets(self,
prev_status_code,
reproduce=False,
lock=lock)
sequence_failed = True
break
if prev_parser_threw_exception:
logger.write_to_main(
"Error: Parser exception occurred during valid sequence re-rendering.\n"
)
sequence_failed = True
break
if resource_error:
logger.write_to_main(
"Error: The resource was left in a Failed state after creation during valid sequence re-rendering.\n"
)
sequence_failed = True
break
# If the previous request is a resource generator and we did not perform an async resource
# creation wait, then wait for the specified duration in order for the backend to have a
# chance to create the resource.
if prev_producer_timing_delay > 0 and prev_request.is_resource_generator(
):
print(
f"Pausing for {prev_producer_timing_delay} seconds, request is a generator..."
)
time.sleep(prev_producer_timing_delay)
# register latest client/server interaction
timestamp_micro = int(time.time() * 10**6)
self.status_codes.append(
status_codes_monitor.RequestExecutionStatus(
timestamp_micro,
prev_request.hex_definition, prev_status_code,
prev_response.has_valid_code(), False))
if sequence_failed:
self.status_codes.append(
status_codes_monitor.RequestExecutionStatus(
int(time.time() * 10**6), request.hex_definition,
RESTLER_INVALID_CODE, False, True))
Monitor().update_status_codes_monitor(self, self.status_codes,
lock)
return RenderedSequence(
failure_info=FailureInformation.SEQUENCE)
# Step B: Dynamic template rendering
# substitute reference placeholders with ressoved values
# for the last request
if not Settings().ignore_dependencies:
rendered_data = self.resolve_dependencies(rendered_data)
# Render candidate value combinations seeking for valid error codes
request._current_combination_id += 1
req_async_wait = Settings().get_max_async_resource_creation_time(
request.request_id)
response = request_utilities.send_request_data(rendered_data)
response_to_parse, resource_error, _ = async_request_utilities.try_async_poll(
rendered_data, response, req_async_wait)
parser_exception_occurred = False
# Response may not exist if there was an error sending the request or a timeout
if parser and response_to_parse:
parser_exception_occurred = not request_utilities.call_response_parser(
parser, response_to_parse, request)
status_code = response.status_code
if not status_code:
return RenderedSequence(None)
self.append_data_to_sent_list(rendered_data,
parser,
response,
max_async_wait_time=req_async_wait)
rendering_is_valid = not parser_exception_occurred\
and not resource_error\
and response.has_valid_code()
# register latest client/server interaction and add to the status codes list
response_datetime = datetime.datetime.now(datetime.timezone.utc)
timestamp_micro = int(response_datetime.timestamp() * 10**6)
self.status_codes.append(
status_codes_monitor.RequestExecutionStatus(
timestamp_micro, request.hex_definition, status_code,
rendering_is_valid, False))
# add sequence's error codes to bug buckets.
if response.has_bug_code():
BugBuckets.Instance().update_bug_buckets(self,
status_code,
lock=lock)
Monitor().update_status_codes_monitor(self, self.status_codes,
lock)
# Register current rendering's status.
if lock is not None:
lock.acquire()
Monitor().update_renderings_monitor(request, rendering_is_valid)
if lock is not None:
lock.release()
if Monitor().remaining_time_budget <= 0 and not postprocessing:
raise TimeOutException("Exceeded Timeout")
if lock is not None:
lock.acquire()
# Deep copying here will try copying anything the class has access
# to including the shared client monitor, which we update in the
# above code block holding the lock, but then we release the
# lock and one thread can be updating while another is copying.
# This is a typlical nasty read after write syncronization bug.
duplicate = copy.deepcopy(self)
if lock is not None:
lock.release()
datetime_format = "%Y-%m-%d %H:%M:%S"
# return a rendered clone if response indicates a valid status code
if rendering_is_valid or Settings().ignore_feedback:
return RenderedSequence(
duplicate,
valid=True,
final_request_response=response,
response_datetime=response_datetime.strftime(
datetime_format))
else:
information = None
if response.has_valid_code():
if parser_exception_occurred:
information = FailureInformation.PARSER
elif resource_error:
information = FailureInformation.RESOURCE_CREATION
elif response.has_bug_code():
information = FailureInformation.BUG
return RenderedSequence(
duplicate,
valid=False,
failure_info=information,
final_request_response=response,
response_datetime=response_datetime.strftime(
datetime_format))
return RenderedSequence(None)
def get_checker_list(req_collection,
fuzzing_requests,
enable_list,
disable_list,
set_enable_first,
custom_checkers,
enable_default_checkers=True):
""" Initializes all of the checkers, sets the appropriate checkers
as enabled/disabled, and returns a list of checker objects
Note: Order may matter for checkers, in the sense that some checkers (like
the namespacechecker) reset the state before fuzzing, while others (like the
use-after-free checker) start operating immediately after the main driver.
Thus, to be safe, we do not want to reorder the checkers.
The checkers (at least in Python 2.7) are added to the list in the order
that they are imported, which is defined in checkers/__init__.py.
InvalidDynamicObjectChecker was put to the back of the checker order,
so it doesn't interfere with the others. It also re-renders all of the
sequences that it needs itself, so it shouldn't be affected by the other
checkers either.
As long as the checkers are not re-ordered there shouldn't be any issue with
skipping some. They don't rely on each other, but it's possible that some
checkers could affect the fuzzing state in such a way that the next checker
could behave incorrectly. For instance, LeakageRule needs to use the
last_rendering_cache from the Fuzz, so we don't want that to be affected by
another checker (this is why it is run first).
@param req_collection: The global request collection
@type req_collection: RequestCollection
@param fuzzing_requests: The collection of requests to fuzz
@type fuzzing_requests: FuzzingRequestCollection
@param enable_list: The user-specified list of checkers to enable
@type enable_list: List[str]
@param disable_list: The user-specified list of checkers to disable
@type disable_list: List[str]
@param set_enable_first: This sets the ordering priority for the cases where
the user specifies the same checker in both lists.
If this is True, set the enabled values first and then
set the disabled values (or vice-versa if False).
@type set_enable_first: Bool
@param custom_checkers: List of paths to custom checker python files
@type custom_checkers: List[str]
@param enable_default_checkers: If set to False, each checker will be disabled by default,
otherwise, checkers will be enabled/disabled based on their
default settings.
@type enable_default_checkers: Bool
@return: List of Checker objects to apply
@rtype : List[Checker]
"""
# Add any custom checkers
for custom_checker_file_path in custom_checkers:
try:
spec = importlib.util.spec_from_file_location(
'custom_checkers', custom_checker_file_path)
checker = importlib.util.module_from_spec(spec)
spec.loader.exec_module(checker)
logger.write_to_main(
f"Loaded custom checker from {custom_checker_file_path}",
print_to_console=True)
except Exception as err:
logger.write_to_main(
f"Failed to load custom checker {custom_checker_file_path}: {err!s}",
print_to_console=True)
sys.exit(-1)
# Initialize the checker subclasses from CheckerBase
available_checkers = [checker(req_collection, fuzzing_requests)\
for checker in checkers.CheckerBase.__subclasses__()]
# Set the first and second lists based on the set_enable_first Bool
if set_enable_first:
first_list = enable_list
second_list = disable_list
first_enable = True
second_enable = False
else:
first_list = disable_list
second_list = enable_list
first_enable = False
second_enable = True
# Convert lists to lowercase for case-insensitive comparisons
first_list = [x.lower() for x in first_list]
second_list = [x.lower() for x in second_list]
if '*' in second_list:
second_list = []
for checker in available_checkers:
second_list.append(checker.friendly_name)
# If the second list (priority list) is set to all,
# do not use the first list
elif '*' in first_list:
first_list = []
for checker in available_checkers:
first_list.append(checker.friendly_name)
# Iterate through each checker and search for its friendly name
# in each list of enabled/disabled
for checker in available_checkers:
if not enable_default_checkers:
checker.enabled = False
if checker.friendly_name in first_list:
checker.enabled = first_enable
if checker.friendly_name in second_list:
checker.enabled = second_enable
return available_checkers
sys.exit(-1)
if settings.no_tokens_in_logs:
logger.no_tokens_in_logs()
if args.replay_log:
try:
logger.create_network_log(logger.LOG_TYPE_REPLAY)
driver.replay_sequence_from_log(args.replay_log,
settings.token_refresh_cmd)
print("Done playing sequence from log")
sys.exit(0)
except NoTokenSpecifiedException:
logger.write_to_main(
"Failed to play sequence from log:\n"
"A valid authorization token was expected.\n"
"Retry with a token refresh script in the settings file or "
"update the request in the replay log with a valid authorization token.",
print_to_console=True)
sys.exit(-1)
except Exception as error:
print(f"Failed to play sequence from log:\n{error!s}")
sys.exit(-1)
# Import grammar from a restler_grammar file
if args.restler_grammar:
try:
req_collection = import_grammar(args.restler_grammar)
req_collection.set_grammar_name(args.restler_grammar)
except Exception as error:
print(f"Cannot import grammar: {error!s}")
sys.exit(-1)
def apply_create_once_resources(fuzzing_requests):
""" Attempts to create all of the resources in the 'create_once' endpoints.
@param fuzzing_requests: The collection of requests to be fuzzed
@type fuzzing_requests: FuzzingRequestCollection
@return: A list of destructors to use to cleanup the create_once resources
@rtype : list(Request)
"""
def exclude_requests(pre_reqs, post_reqs):
# Exclude any requests that produce or destroy the create_once endpoint
for req_i in pre_reqs:
fuzzing_requests.exclude_preprocessing_request(req_i)
for req_i in post_reqs:
fuzzing_requests.exclude_postprocessing_request(req_i)
create_once_endpoints = Settings().create_once_endpoints
if not create_once_endpoints:
return
logger.create_network_log(logger.LOG_TYPE_PREPROCESSING)
destructors = set()
exclude_reqs = set()
request_count = 0
logger.write_to_main("Rendering for create-once resources:\n")
# Iterate through each 'create_once' endpoint
for endpoint in create_once_endpoints:
# Verify that the endpoint exists in the request collection
if endpoint in GrammarRequestCollection().request_id_collection:
# The create_once resource generator
resource_gen_req = None
# Iterate through each of the requests that contain the create_once endpoint
for req in GrammarRequestCollection(
).request_id_collection[endpoint]:
if req not in fuzzing_requests:
logger.write_to_main(
"Warning: Create-once endpoint is not a request in the fuzzing list\n",
True)
break
if not resource_gen_req and req.is_resource_generator():
resource_gen_req = req
# Compute the sequence necessary to create the create_once resource
req_list = driver.compute_request_goal_seq(
resource_gen_req, fuzzing_requests)
logger.write_to_main(
f"{formatting.timestamp()}: Endpoint - {resource_gen_req.endpoint_no_dynamic_objects}"
)
logger.write_to_main(
f"{formatting.timestamp()}: Hex Def - {resource_gen_req.method_endpoint_hex_definition}"
)
create_once_seq = sequences.Sequence(req_list)
renderings = create_once_seq.render(
GrammarRequestCollection().candidate_values_pool,
None,
preprocessing=True)
# Make sure we were able to successfully create the create_once resource
if not renderings.valid:
logger.write_to_main(
f"{formatting.timestamp()}: Rendering INVALID")
exclude_requests(exclude_reqs, destructors)
raise FailedToCreateResource(destructors)
logger.write_to_main(
f"{formatting.timestamp()}: Rendering VALID")
logger.format_rendering_stats_definition(
resource_gen_req,
GrammarRequestCollection().candidate_values_pool)
if Settings().in_smoke_test_mode():
resource_gen_req.stats.request_order = 'Preprocessing'
resource_gen_req.stats.valid = 1
resource_gen_req.stats.status_code = renderings.final_request_response.status_code
resource_gen_req.stats.status_text = renderings.final_request_response.status_text
resource_gen_req.stats.sample_request.set_request_stats(
renderings.sequence.sent_request_data_list[-1].
rendered_data)
resource_gen_req.stats.sample_request.set_response_stats(
renderings.final_request_response,
renderings.final_response_datetime)
if req.is_destructor():
# Add destructors to the destructor list that will be returned
destructors.add(req)
# Only continue processing if a resource generator was actually found for this endpoint
if not resource_gen_req:
continue
request_count += len(req_list)
# Get the set of all dynamic object names in the endpoint
var_names = resource_gen_req.consumes.union(
resource_gen_req.produces)
# This dictionary will map dynamic object names to the values created during
# this preprocessing create-once step.
dynamic_object_values = {}
for name in var_names:
dynamic_object_values[name] = dependencies.get_variable(name)
# Iterate through the entire request collection, searching for requests that include
# the create_once resource. We want to "lock" the resources in these requests with
# the dynamic object values that were created during this preprocessing step.
for req_i in fuzzing_requests:
# Set the variables in any requests whose consumers were produced
# by the create_once resource generator
if resource_gen_req.produces & req_i.consumes:
req_i.set_id_values_for_create_once_dynamic_objects(
dynamic_object_values, renderings)
# Exclude any requests that produce the create_once object(s)
if resource_gen_req.produces & req_i.produces:
exclude_reqs.add(req_i)
else:
exclude_requests(exclude_reqs, destructors)
raise InvalidCreateOnce(destructors)
exclude_requests(exclude_reqs, destructors)
# Reset all of the dynamic object values that were just created
dependencies.reset_tlb()
# Reset the garbage collector, so it doesn't delete any of the resources that were just created
dependencies.set_saved_dynamic_objects()
logger.print_request_rendering_stats(
GrammarRequestCollection().candidate_values_pool, fuzzing_requests,
Monitor(), request_count, logger.PREPROCESSING_GENERATION, None)
# Return the list of destructors that were removed from the request collection.
# These will be used to cleanup the create_once resources created during preprocessing.
return list(destructors)
def _print_req_not_found():
logger.write_to_main(
"Request from grammar does not exist in the Request Collection!\n"
f"{method} {endpoint}\n",\
print_to_console=True
)
if os.path.isabs(custom_mutations_paths[endpoint]):
path = custom_mutations_paths[endpoint]
else:
# If custom dictionary path is not an absolute path, make it relative to the grammar
path = os.path.join(os.path.dirname(args.restler_grammar), custom_mutations_paths[endpoint])
with open(path, 'r') as mutations:
per_endpoint_custom_mutations[endpoint] = json.load(mutations)
except Exception as error:
print(f"Cannot import custom mutations: {error!s}")
sys.exit(-1)
try:
req_collection.set_custom_mutations(custom_mutations, per_endpoint_custom_mutations)
except UnsupportedPrimitiveException as primitive:
logger.write_to_main("Error in mutations dictionary.\n"
f"Unsupported primitive type defined: {primitive!s}",
print_to_console=True)
sys.exit(-1)
except InvalidDictPrimitiveException as err:
logger.write_to_main("Error in mutations dictionary.\n"
"Dict type primitive was specified as another type.\n"
f"{err!s}",
print_to_console=True)
sys.exit(-1)
if settings.token_refresh_cmd:
req_collection.candidate_values_pool.set_candidate_values(
{
'restler_refreshable_authentication_token':
{
'token_refresh_cmd': settings.token_refresh_cmd,
