def __init__(self, connection_settings):
""" Initializes a socket object using low-level python socket objects.
@param connection_settings: The connection settings for this socket
@type connection_settings: ConnectionSettings
@return: None
@rtype : None
"""
self._request_throttle_sec = (float)(Settings().request_throttle_ms/1000.0)\
if Settings().request_throttle_ms else None
self.connection_settings = connection_settings
try:
self._sock = None
host = Settings().host
target_ip = self.connection_settings.target_ip or host
target_port = self.connection_settings.target_port
if Settings().use_test_socket:
self._sock = TestSocket(Settings().test_server)
elif self.connection_settings.use_ssl:
context = ssl.create_default_context()
with socket.create_connection((target_ip, target_port
or 443)) as sock:
self._sock = context.wrap_socket(sock,
server_hostname=host)
else:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._sock.connect((target_ip, target_port or 80))
except Exception as error:
raise TransportLayerException(
f"Exception Creating Socket: {error!s}")
def replay_sequence_from_log(replay_log_filename, token_refresh_cmd):
""" Replays a sequence of requests from a properly formed log file
@param replay_log_filename: The log's filename
@type replay_log_filename: Str
@param token_refresh_cmd: The command to create an authorization token
@type token_refresh_cmd: Str
@return: None
@rtype : None
"""
log_file = open(replay_log_filename, "r")
file_lines = log_file.readlines()
send_data = []
for line in file_lines:
line = line.strip()
# Check for comment or empty line
if line:
if line.startswith(logger.REPLAY_REQUEST_INDICATOR):
# Clean up the request string before continuing
line = line.lstrip(logger.REPLAY_REQUEST_INDICATOR)
line = line.rstrip('\n')
line = line.replace('\\r', '\r')
line = line.replace('\\n', '\n')
if not Settings().host:
# Extract hostname from request
hostname = get_hostname_from_line(line)
if hostname is None:
raise Exception("Host not found in request. The replay log may be corrupted.")
Settings().set_hostname(hostname)
# Append the request data to the list
# None is for the parser, which does not currently run during replays.
send_data.append(sequences.SentRequestData(line, None))
elif line.startswith(logger.BUG_LOG_NOTIFICATION_ICON):
line = line.lstrip(logger.BUG_LOG_NOTIFICATION_ICON)
if line.startswith('producer_timing_delay'):
if send_data:
# Add the producer timing delay to the most recent request data
send_data[-1].producer_timing_delay = int(line.lstrip('producer_timing_delay '))
if line.startswith('max_async_wait_time'):
if send_data:
# Add the max async wait time to the most recent request data
send_data[-1].max_async_wait_time = int(line.lstrip('max_async_wait_time '))
sequence = sequences.Sequence()
sequence.set_sent_requests_for_replay(send_data)
if token_refresh_cmd:
# Set the authorization tokens in the data
execute_token_refresh_cmd(token_refresh_cmd)
# Send the requests
sequence.replay_sequence()
def send_request_data(rendered_data):
""" Helper that sends a request's rendered data to the server
and parses its response.
@param rendered_data: The data to send to the server
@type rendered_data: Str
@return: The response from the server
@rtype : HttpResponse
"""
# Set max retries and retry sleep time to be used in case
# a status code from the retry list is encountered.
MAX_RETRIES = 5
RETRY_SLEEP_SEC = 5
RETRY_CODES = ['409', '429']
num_retries = 0
while num_retries < MAX_RETRIES:
try:
# Establish connection to server
sock = HttpSock(Settings().connection_settings)
except TransportLayerException as error:
_RAW_LOGGING(str(error))
return HttpResponse()
# Send the request and receive the response
success, response = sock.sendRecv(
rendered_data,
Settings().max_request_execution_time)
status_code = response.status_code
if status_code and status_code in RESTLER_BUG_CODES:
return response
if not success or not status_code:
_RAW_LOGGING(response.to_str)
return HttpResponse()
if status_code in RETRY_CODES:
time.sleep(RETRY_SLEEP_SEC)
num_retries += 1
if num_retries < MAX_RETRIES:
_RAW_LOGGING("Retrying request")
continue
else:
return response
return response
def _send_request(self, parser, rendered_data):
""" Send a request and invoke the response parser.
@param parser: A parser to parse the data
@type parser: Function pointer
@param rendered_data: The request's rendered data to send
@type rendered_data: Str
@return: The response from the server
@rtype : HttpResponse
"""
try:
sock = messaging.HttpSock(self._connection_settings)
except TransportLayerException as error:
RAW_LOGGING(f"{error!s}")
# connection failed
return HttpResponse()
success, response = sock.sendRecv(
rendered_data,
req_timeout_sec=Settings().max_request_execution_time)
if not success:
RAW_LOGGING(response.to_str)
Monitor().increment_requests_count(self.__class__.__name__)
return response
def sendRecv(self, message, req_timeout_sec=600):
""" Sends a specified request to the server and waits for a response
@param message: Message to be sent.
@type message : Str
@param req_timeout_sec: The time, in seconds, to wait for request to complete
@type req_timeout_sec : Int
@return:
False if failure, True if success
Response if True returned, Error if False returned
@rtype : Tuple (Bool, String)
"""
try:
self._sendRequest(message)
if not Settings().use_test_socket:
response = HttpResponse(self._recvResponse(req_timeout_sec))
else:
response = self._sock.recv()
RAW_LOGGING(f'Received: {response.to_str!r}\n')
return (True, response)
except TransportLayerException as error:
response = HttpResponse(str(error).strip('"\''))
if 'timed out' in str(error):
response._status_code = TIMEOUT_CODE
RAW_LOGGING(
f"Reached max req_timeout_sec of {req_timeout_sec}.")
elif self._contains_connection_closed(str(error)):
response._status_code = CONNECTION_CLOSED_CODE
RAW_LOGGING(f"{error!s}")
return (False, response)
finally:
self._closeSocket()
def __init__(self, req_collection, fuzzing_requests, enabled=False):
""" Abstract class constructor
@param req_collection: The shared request collection
@type req_collection: RequestCollection
@param fuzzing_requests: The collection of requests to fuzz
@type fuzzing_requests: FuzzingRequestCollection
@param enabled: Set to True to enable this checker by default when fuzzing
@type enabled: Bool
"""
self._checker_log = CheckerLog(self.__class__.__name__)
self._req_collection = req_collection
self._fuzzing_requests = fuzzing_requests
self._connection_settings = Settings().connection_settings
self._enabled = enabled
self._friendly_name = self.__class__.__name__[:-len("Checker")].lower()
self._mode = Settings().get_checker_arg(self._friendly_name,
'mode') or 'normal'
def has_bug_code(self):
""" Returns True if the status code is considered a bug
@return: True if the status code is considered a bug
@rtype : Bool
"""
if self._status_code:
if Settings().custom_non_bug_codes:
# All codes except the ones in the custom_non_bug_codes list should be flagged as bugs.
# Hence, return False only if the status code exists in the list.
for code in Settings().custom_non_bug_codes:
if re.match(code, self._status_code):
return False
else:
return True
if self._status_code.startswith('5'):
return True
for code in Settings().custom_bug_codes:
if re.match(code, self._status_code):
return True
return False
def _create_fuzzable_dates(self):
""" Creates dates for future and past, which can be added to a list
of restler_fuzzable_datetime candidate values
@return: None
@rtype : None
"""
today = datetime.datetime.today()
# Make sure we add enough days to account for a long fuzzing run
days_to_add = datetime.timedelta(days=(Settings().time_budget / 24) +
1)
future = today + days_to_add
self._future_date = future.strftime(PAYLOAD_DATE_FORMAT)
oneday = datetime.timedelta(days=1)
yesterday = today - oneday
self._past_date = yesterday.strftime(PAYLOAD_DATE_FORMAT)
def __init__(self, req_collection, fuzzing_monitor, interval):
""" Uses requests from @param req_collection to garbage collect, i.e.,
try to periodically try deleting dynamic objects.
@param req_collection: The requests collection.
@type req_collection: RequestCollection class object.
@param fuzzing_monitor: The global monitor for the fuzzing run
@type fuzzing_monitor: FuzzingMonitor
@param interval: The interval after which to restart garbage collection.
@param interval: Int
@return: None
@rtype : None
"""
threading.Thread.__init__(self)
self._interval = interval
self._dyn_objects_cache_size = Settings().dyn_objects_cache_size
self.req_collection = req_collection
self.monitor = fuzzing_monitor
global dyn_objects_cache_lock
dyn_objects_cache_lock = multiprocessing.Lock()
self.dyn_objects_cache_lock = dyn_objects_cache_lock
global dyn_objects_cache
self.dyn_objects_cache = dyn_objects_cache
global saved_dyn_objects
self.saved_dyn_objects = saved_dyn_objects
self._destructor_types = []
# This is a buffer of all overflowing dynamic objects created during the
# lifetime of fuzzing.
self.overflowing = {}
self._finishing = False
self._cleanup_event = threading.Event()
def _render_and_send_data(self, seq, request, check_async=True):
""" Helper that renders data for a request, sends the request to the server,
and then adds that rendered data and its response to a sequence's sent-request-data
list. This is here so that checkers can send their own requests without needing to
run through the sequences.render function. Adding the data to the sent-request-data
list is required when replaying a sequence, which occurs when a bug is detected.
@param seq: The sequence to append the rendered data to
@type seq: Sequence
@param request: The request to render and append
@type request: Request
@param check_async: If set to True (default), the function will check for resources
that are created asynchronously and wait for them if so.
@type check_async: Boolean
@return: Tuple containing the response received after sending the request and the response
that should be parsed. The response_to_parse will differ from response only if
the response_to_parse was from a GET request that followed an asynchronous resource creation.
@rtype : Tuple(HttpResponse, HttpResponse)
"""
rendered_data, parser = request.render_current(
self._req_collection.candidate_values_pool)
rendered_data = seq.resolve_dependencies(rendered_data)
response = self._send_request(parser, rendered_data)
response_to_parse = response
async_wait = Settings().get_max_async_resource_creation_time(
request.request_id)
if check_async:
response_to_parse, _, _ = async_request_utilities.try_async_poll(
rendered_data, response, async_wait)
request_utilities.call_response_parser(parser, response_to_parse)
seq.append_data_to_sent_list(rendered_data,
parser,
response,
producer_timing_delay=0,
max_async_wait_time=async_wait)
return response, response_to_parse
def __init__(self, definition=[], requestId=None):
""" Initialize a request object by assigning a definition and
internally constructing the necessary constraints.
@param definition: List of restler primitives and directives
corresponding to what we call "restler grammar".
@type definition: List
@param requestId: The request's ID string
@type requestId: Str
@return: None
@rtype : None
"""
self._current_combination_id = 0
self._total_feasible_combinations = 0
self._hex_definition = 0
self._method_endpoint_hex_definition = 0
self._request_id = 0
self._endpoint_no_dynamic_objects = requestId
self._definition = definition
self._examples = None
self._body_schema = None
self._consumes = set()
self._produces = set()
self._set_constraints()
self._create_once_requests = []
# Check for empty request before assigning ids
if self._definition:
self._set_hex_definitions(requestId)
else:
raise EmptyRequestException
if Settings().in_smoke_test_mode():
self.stats = SmokeTestStats()
def generate_sequences(fuzzing_requests, checkers, fuzzing_jobs=1):
""" Implements core restler algorithm.
@param fuzzing_requests: The collection of requests that will be fuzzed
@type fuzzing_requests: FuzzingRequestCollection
@param checkers: The list of checkers to apply
@type checkers: list[Checker]
@param fuzzing_jobs: Optional number of fuzzing jobs for parallel fuzzing.
Default value passed is one (sequential fuzzing).
@type fuzzing_jobs: Int
@return: None
@rtype : None
"""
if not fuzzing_requests.size:
return
logger.create_network_log(logger.LOG_TYPE_TESTING)
fuzzing_mode = Settings().fuzzing_mode
max_len = Settings().max_sequence_length
if fuzzing_mode == 'directed-smoke-test':
return generate_sequences_directed_smoketest(fuzzing_requests,
checkers)
if fuzzing_jobs > 1:
render = render_parallel
global_lock = multiprocessing.Lock()
fuzzing_pool = ThreadPool(fuzzing_jobs)
else:
global_lock = None
fuzzing_pool = None
render = render_sequential
should_stop = False
timeout_reached = False
seq_collection_exhausted = False
num_total_sequences = 0
while not should_stop:
seq_collection = [sequences.Sequence()]
# Only for bfs: If any checkpoint file is available, load state of
# latest generation. Note that it only makes sense to use checkpoints
# for the bfs exploration method, since it is the only systemic and
# exhaustive method.
min_len = 0
if fuzzing_mode == 'bfs':
req_collection = GrammarRequestCollection()
monitor = Monitor()
req_collection, seq_collection, fuzzing_requests, monitor, min_len =\
saver.load(req_collection, seq_collection, fuzzing_requests, monitor)
requests.GlobalRequestCollection.Instance(
)._req_collection = req_collection
fuzzing_monitor.FuzzingMonitor.__instance = monitor
# Repeat external loop only for random walk
if fuzzing_mode != 'random-walk':
should_stop = True
# Initialize fuzzing schedule
fuzzing_schedule = {}
logger.write_to_main(f"Setting fuzzing schemes: {fuzzing_mode}")
for length in range(min_len, max_len):
fuzzing_schedule[length] = fuzzing_mode
# print(" - {}: {}".format(length + 1, fuzzing_schedule[length]))
# print general request-related stats
logger.print_req_collection_stats(
fuzzing_requests,
GrammarRequestCollection().candidate_values_pool)
generation = 0
for length in range(min_len, max_len):
# we can set this without locking, since noone else writes (main
# driver is single-threaded) and every potential worker will just
# read-access this value.
generation = length + 1
fuzzing_mode = fuzzing_schedule[length]
# extend sequences with new request templates
seq_collection = extend(seq_collection, fuzzing_requests,
global_lock)
print(f"{formatting.timestamp()}: Generation: {generation} ")
logger.write_to_main(
f"{formatting.timestamp()}: Generation: {generation} / "
f"Sequences Collection Size: {len(seq_collection)} "
f"(After {fuzzing_schedule[length]} Extend)")
# render templates
try:
seq_collection_exhausted = False
seq_collection = render(seq_collection, fuzzing_pool, checkers,
generation, global_lock)
except TimeOutException:
logger.write_to_main("Timed out...")
timeout_reached = True
seq_collection_exhausted = True
# Increase fuzzing generation after timeout because the code
# that does it would have never been reached. This is done so
# the previous generation's test summary is logged correctly.
Monitor().current_fuzzing_generation += 1
except ExhaustSeqCollectionException:
logger.write_to_main("Exhausted collection...")
seq_collection = []
seq_collection_exhausted = True
logger.write_to_main(
f"{formatting.timestamp()}: Generation: {generation} / "
f"Sequences Collection Size: {len(seq_collection)} "
f"(After {fuzzing_schedule[length]} Render)")
# saving latest state
saver.save(GrammarRequestCollection(), seq_collection,
fuzzing_requests, Monitor(), generation)
# Print stats for iteration of the current generation
logger.print_generation_stats(GrammarRequestCollection(),
Monitor(), global_lock)
num_total_sequences += len(seq_collection)
logger.print_request_rendering_stats(
GrammarRequestCollection().candidate_values_pool,
fuzzing_requests, Monitor(),
Monitor().num_fully_rendered_requests(
fuzzing_requests.all_requests), generation, global_lock)
if timeout_reached or seq_collection_exhausted:
if timeout_reached:
should_stop = True
break
logger.write_to_main("--\n")
if fuzzing_pool is not None:
fuzzing_pool.close()
fuzzing_pool.join()
return num_total_sequences
def extend(seq_collection, fuzzing_requests, lock):
""" Extends each sequence currently present in collection by any request
from request collection whose dependencies can be resolved if appended at
the end of the target sequence.
@param seq_collection: List of sequences in sequence collection.
@type seq_collection: List
@param fuzzing_requests: The collection of requests to fuzz.
@type fuzzing_requests: FuzzingRequestCollection.
@param lock: Lock object used for sync of more than one fuzzing jobs.
@type lock: thread.Lock object
@return: The list of newly enxtended sequences.
@rtype : List
"""
prev_len = len(seq_collection)
# The functions that access the monitor of renderings (e.g.,
# "is_fully_rendered_request" and "num_fully_rendered_requests") answer
# based on the latest _completed_ generation and the internal
# counter that tracks the latest completed fuzzing generation is increased
# after the end of @function render. However, inside the driver main-loop we
# first run @function extend (since initially we start by an empty
# sequence) and then run @function render, and thus, we need to temporarily
# increase the generation counter in order to get a proper behaviour
# when invoking "is_fully_rendered_request" in here after the first iteration
# of the main-loop.
Monitor().current_fuzzing_generation += 1
for req in fuzzing_requests:
for i in range(prev_len):
seq = seq_collection[i]
# Extend sequence collection by adding requests that have
# valid dependencies and skip the rest
if not validate_dependencies(req, seq)\
and not Settings().ignore_dependencies:
continue
req_copy = copy.copy(req)
req_copy._current_combination_id = 0
if seq.is_empty_sequence():
new_seq = sequences.Sequence(req_copy)
else:
new_seq = seq + sequences.Sequence(req_copy)
seq_collection.append(new_seq)
# Append each request to exactly one sequence
if Settings().fuzzing_mode in ['bfs-fast', 'bfs-minimal']:
break
# See comment above...
Monitor().current_fuzzing_generation -= 1
# In case of random walk, truncate sequence collection to
# one randomly selected sequence
if Settings().fuzzing_mode == 'random-walk':
if len(seq_collection) > 0:
rand_int = random.randint(prev_len, len(seq_collection) - 1)
return seq_collection[rand_int:rand_int + 1]
else:
return []
# Drop previous generation and keep current extended generation
return seq_collection[prev_len:]
def render_one(seq_collection, ith, checkers, generation, global_lock):
""" Render ith sequence from sequence collection.
@param seq_collection: List of sequences in sequence collection.
@type seq_collection: List
@param ith: The position of the target sequence (to be rendered) in the
sequence collection.
@type ith: Int
@param checkers: The list of checkers to apply
@type checkers: list[Checker]
@param generation: The fuzzing generation
@type generation: Int
@param global_lock: Lock object used for sync of more than one fuzzing jobs.
@type global_lock: thread.Lock object
@return: The list of sequences with valid renderings.
@rtype : List
Note: Try ith sequence's template with all posible primitive type value
combinations and return only renderings (combinations of primitive type
values) that lead to valid error codes. We keep track of the order of the
current sequence in the collection using "ith" argument for logging
purposes.
"""
# Log memory consumption every hour.
n_minutes = 60
# Static variable used for keeping track of the last time memory consumption was printed
render_one.last_memory_consumption_check = getattr(
render_one, 'last_memory_consumption_check', int(time.time()))
if int(time.time()) - render_one.last_memory_consumption_check > (
n_minutes * 60):
logger.print_memory_consumption(GrammarRequestCollection(), Monitor(),
Settings().fuzzing_mode, generation)
render_one.last_memory_consumption_check = int(time.time())
candidate_values_pool = GrammarRequestCollection().candidate_values_pool
current_seq = seq_collection[ith]
current_seq.seq_i = ith
valid_renderings = []
# Try to find one valid rendering.
n_invalid_renderings = 0
while True:
# Render on a sequence instance will internally iterate over possible
# renderings of current sequence until a valid or an invalid combination
# of values for its primitive types is found -- internal iteration may
# skip some renderings (that are marked to be skipped according to past
# failures) -- that's why we put everything in a while.
renderings = current_seq.render(candidate_values_pool, global_lock)
# Note that this loop will keep running as long as we hit invalid
# renderings and we will end up reapplying the leakage rule a billion
# times for very similar 404s. To control this, when in bfs-cheap, we
# apply the checkers only on the first invalid rendering.
if Settings().fuzzing_mode not in ['bfs-cheap', 'bfs-minimal']\
or renderings.valid or n_invalid_renderings < 1:
apply_checkers(checkers, renderings, global_lock)
# If renderings.sequence is None it means there is nothing left to render.
if renderings.valid or renderings.sequence is None:
break
# This line will only be reached only if we have an invalid rendering.
n_invalid_renderings += 1
# for random-walk and cheap fuzzing, one valid rendering is enough.
if Settings().fuzzing_mode in ['random-walk', 'bfs-cheap', 'bfs-minimal']:
if renderings.valid:
valid_renderings.append(renderings.sequence)
# bfs needs to be exhaustive to provide full grammar coverage
elif Settings().fuzzing_mode in ['bfs', 'bfs-fast']:
# This loop will iterate over possible remaining renderings of the
# current sequence.
while renderings.sequence is not None:
if renderings.valid:
valid_renderings.append(renderings.sequence)
renderings = current_seq.render(candidate_values_pool, global_lock)
apply_checkers(checkers, renderings, global_lock)
else:
print("Unsupported fuzzing_mode:", Settings().fuzzing_mode)
assert False
return valid_renderings
def apply_destructors(self, destructors, max_aged_objects=100):
""" Background task trying to delete evicted objects that are in
@overflowing dictionary.
@param overflowing: Dictionary of overflowing objects. We need to issue
deletes to all these objects
@type overflowing: Dict
@param destructors: The Request class objects required to delete.
@type destructors: Dict
@param max_aged_objects: Maximum number of objects we allow to age
and will retry to delete (since delete
is idempotent this is fine).
@type max_aged_objects: Int
@return: None
@rtype : None
NOTE: This function is invoked without any lock since overflowing
objects are already dead (not referenced by anything) and are just
aging here.
"""
if not self.overflowing:
return
from engine.errors import TransportLayerException
from engine.transport_layer import messaging
from utils.logger import raw_network_logging as RAW_LOGGING
from utils.logger import garbage_collector_logging as CUSTOM_LOGGING
# For each object in the overflowing area, whose destructor is
# available, render the corresponding request, send the request,
# and then check the status code. If the resource has been determined
# to be removed, delete the object from the overflow area.
# At the end keep track of only up to @param max_aged_objects
# remaining objects.
for type in destructors:
destructor = destructors[type]
deleted_list = []
if self.overflowing[type]:
CUSTOM_LOGGING("{}: Trying garbage collection of * {} * objects".\
format(formatting.timestamp(), len(self.overflowing[type])))
CUSTOM_LOGGING(f"{type}: {self.overflowing[type]}")
# Iterate in reverse to give priority to newest resources
for value in reversed(self.overflowing[type]):
rendered_data, _ = destructor.\
render_current(self.req_collection.candidate_values_pool)
# replace dynamic parameters
fully_rendered_data = str(rendered_data)
fully_rendered_data = fully_rendered_data.replace(
RDELIM + type + RDELIM, value)
if fully_rendered_data:
try:
# Establish connection to the server
sock = messaging.HttpSock(
Settings().connection_settings)
except TransportLayerException as error:
RAW_LOGGING(f"{error!s}")
return
# Send the request and receive the response
success, response = sock.sendRecv(fully_rendered_data)
if success:
self.monitor.increment_requests_count('gc')
else:
RAW_LOGGING(response.to_str)
# Check to see if the DELETE operation is complete
try:
if response.status_code in DELETED_CODES:
deleted_list.append(value)
except Exception:
pass
# Remove deleted items from the to-delete cache
for value in deleted_list:
self.overflowing[type].remove(value)
self.overflowing[type] = self.overflowing[type][-max_aged_objects:]
def apply_create_once_resources(fuzzing_requests):
""" Attempts to create all of the resources in the 'create_once' endpoints.
@param fuzzing_requests: The collection of requests to be fuzzed
@type fuzzing_requests: FuzzingRequestCollection
@return: A list of destructors to use to cleanup the create_once resources
@rtype : list(Request)
"""
def exclude_requests(pre_reqs, post_reqs):
# Exclude any requests that produce or destroy the create_once endpoint
for req_i in pre_reqs:
fuzzing_requests.exclude_preprocessing_request(req_i)
for req_i in post_reqs:
fuzzing_requests.exclude_postprocessing_request(req_i)
create_once_endpoints = Settings().create_once_endpoints
if not create_once_endpoints:
return
logger.create_network_log(logger.LOG_TYPE_PREPROCESSING)
destructors = set()
exclude_reqs = set()
request_count = 0
logger.write_to_main("Rendering for create-once resources:\n")
# Iterate through each 'create_once' endpoint
for endpoint in create_once_endpoints:
# Verify that the endpoint exists in the request collection
if endpoint in GrammarRequestCollection().request_id_collection:
# The create_once resource generator
resource_gen_req = None
# Iterate through each of the requests that contain the create_once endpoint
for req in GrammarRequestCollection(
).request_id_collection[endpoint]:
if req not in fuzzing_requests:
logger.write_to_main(
"Warning: Create-once endpoint is not a request in the fuzzing list\n",
True)
break
if not resource_gen_req and req.is_resource_generator():
resource_gen_req = req
# Compute the sequence necessary to create the create_once resource
req_list = driver.compute_request_goal_seq(
resource_gen_req, fuzzing_requests)
logger.write_to_main(
f"{formatting.timestamp()}: Endpoint - {resource_gen_req.endpoint_no_dynamic_objects}"
)
logger.write_to_main(
f"{formatting.timestamp()}: Hex Def - {resource_gen_req.method_endpoint_hex_definition}"
)
create_once_seq = sequences.Sequence(req_list)
renderings = create_once_seq.render(
GrammarRequestCollection().candidate_values_pool,
None,
preprocessing=True)
# Make sure we were able to successfully create the create_once resource
if not renderings.valid:
logger.write_to_main(
f"{formatting.timestamp()}: Rendering INVALID")
exclude_requests(exclude_reqs, destructors)
raise FailedToCreateResource(destructors)
logger.write_to_main(
f"{formatting.timestamp()}: Rendering VALID")
logger.format_rendering_stats_definition(
resource_gen_req,
GrammarRequestCollection().candidate_values_pool)
if Settings().in_smoke_test_mode():
resource_gen_req.stats.request_order = 'Preprocessing'
resource_gen_req.stats.valid = 1
resource_gen_req.stats.status_code = renderings.final_request_response.status_code
resource_gen_req.stats.status_text = renderings.final_request_response.status_text
resource_gen_req.stats.sample_request.set_request_stats(
renderings.sequence.sent_request_data_list[-1].
rendered_data)
resource_gen_req.stats.sample_request.set_response_stats(
renderings.final_request_response,
renderings.final_response_datetime)
if req.is_destructor():
# Add destructors to the destructor list that will be returned
destructors.add(req)
# Only continue processing if a resource generator was actually found for this endpoint
if not resource_gen_req:
continue
request_count += len(req_list)
# Get the set of all dynamic object names in the endpoint
var_names = resource_gen_req.consumes.union(
resource_gen_req.produces)
# This dictionary will map dynamic object names to the values created during
# this preprocessing create-once step.
dynamic_object_values = {}
for name in var_names:
dynamic_object_values[name] = dependencies.get_variable(name)
# Iterate through the entire request collection, searching for requests that include
# the create_once resource. We want to "lock" the resources in these requests with
# the dynamic object values that were created during this preprocessing step.
for req_i in fuzzing_requests:
# Set the variables in any requests whose consumers were produced
# by the create_once resource generator
if resource_gen_req.produces & req_i.consumes:
req_i.set_id_values_for_create_once_dynamic_objects(
dynamic_object_values, renderings)
# Exclude any requests that produce the create_once object(s)
if resource_gen_req.produces & req_i.produces:
exclude_reqs.add(req_i)
else:
exclude_requests(exclude_reqs, destructors)
raise InvalidCreateOnce(destructors)
exclude_requests(exclude_reqs, destructors)
# Reset all of the dynamic object values that were just created
dependencies.reset_tlb()
# Reset the garbage collector, so it doesn't delete any of the resources that were just created
dependencies.set_saved_dynamic_objects()
logger.print_request_rendering_stats(
GrammarRequestCollection().candidate_values_pool, fuzzing_requests,
Monitor(), request_count, logger.PREPROCESSING_GENERATION, None)
# Return the list of destructors that were removed from the request collection.
# These will be used to cleanup the create_once resources created during preprocessing.
return list(destructors)
def render_iter(self, candidate_values_pool, skip=0, preprocessing=False):
""" This is the core method that renders values combinations in a
request template. It basically is a generator which lazily iterates over
a pool of possible combination of values that fit the template of the
requst. Static primitive types (such as, string or delimiters) are
picked from @param candidate_values_pool; dynamic primitive types
(such as, uuid4) are generated "fresh" every time the generator yields
with the help of @method resolve_dynamic_primitives.
@param candidate_values_pool: The pool of values for primitive types.
@type candidate_values_pool: Dict
@param skip: Number of combinations to skip (i.e., not render). This is
useful when we use a request as an "ingredient" of a
sequence and we are only interested in using a specific
rendering that, we know, leads to a valid rendering with
a desired status code response.
@type skip: Int
@param preprocessing: Set to True if this rendering is happening during preprocessing
@type preprocessing: Bool
@return: (rendered request's payload, response's parser function)
@rtype : (Str, Function Pointer)
"""
def _raise_dict_err(type, tag):
logger.write_to_main(
f"Error for request {self.method} {self.endpoint_no_dynamic_objects}.\n"
f"{type} exception: {tag} not found.\n"
"Make sure you are using the dictionary created during compilation.",
print_to_console=True)
raise InvalidDictionaryException
def _handle_exception(type, tag, err):
logger.write_to_main(
f"Exception when rendering request {self.method} {self.endpoint_no_dynamic_objects}.\n"
f"Type: {type}. Tag: {tag}.\n"
f" Exception: {err!s}",
print_to_console=True)
raise InvalidDictionaryException
if not candidate_values_pool:
print("Candidate values pool empty")
print(self._definition)
yield []
return
definition = self.definition
if not definition:
yield []
return
parser = None
# If request had post_send metadata, register parsers etc.
if bool(self.metadata) and 'post_send' in self.metadata\
and 'parser' in self.metadata['post_send']:
parser = self.metadata['post_send']['parser']
fuzzable = []
for request_block in definition:
primitive_type = request_block[0]
default_val = request_block[1]
quoted = request_block[2]
values = []
# Handling dynamic primitives that need fresh rendering every time
if primitive_type == primitives.FUZZABLE_UUID4:
values = [primitives.restler_fuzzable_uuid4]
# Handle enums that have a list of values instead of one default val
elif primitive_type == primitives.FUZZABLE_GROUP:
values = list(request_block[1])
# Handle static whose value is the field name
elif primitive_type == primitives.STATIC_STRING:
values = [request_block[1]]
# Handle multipart form data
elif primitive_type == primitives.FUZZABLE_MULTIPART_FORMDATA:
try:
current_fuzzable_values = candidate_values_pool.\
get_candidate_values(primitive_type, request_id=self._request_id, tag=default_val, quoted=quoted)
values = [
multipart_formdata.render(current_fuzzable_values)
]
except primitives.CandidateValueException:
_raise_dict_err(primitive_type, default_val)
except Exception as err:
_handle_exception(primitive_type, default_val, err)
# Handle custom (user defined) static payload
elif primitive_type == primitives.CUSTOM_PAYLOAD:
try:
current_fuzzable_values = candidate_values_pool.\
get_candidate_values(primitive_type, request_id=self._request_id, tag=default_val, quoted=quoted)
# handle case where custom payload have more than one values
if isinstance(current_fuzzable_values, list):
values = current_fuzzable_values
else:
values = [current_fuzzable_values]
except primitives.CandidateValueException:
_raise_dict_err(primitive_type, default_val)
except Exception as err:
_handle_exception(primitive_type, default_val, err)
# Handle custom (user defined) static payload on header (Adds \r\n)
elif primitive_type == primitives.CUSTOM_PAYLOAD_HEADER:
try:
current_fuzzable_values = candidate_values_pool.\
get_candidate_values(primitive_type, request_id=self._request_id, tag=default_val, quoted=quoted)
# handle case where custom payload have more than one values
if isinstance(current_fuzzable_values, list):
values = current_fuzzable_values
else:
values = [current_fuzzable_values]
if values:
values = list(
map(
lambda x: "{}: {}\r\n".\
format(default_val, x),
values
)
)
except primitives.CandidateValueException:
_raise_dict_err(primitive_type, default_val)
except Exception as err:
_handle_exception(primitive_type, default_val, err)
# Handle custom (user defined) static payload with uuid4 suffix
elif primitive_type == primitives.CUSTOM_PAYLOAD_UUID4_SUFFIX:
try:
current_fuzzable_value = candidate_values_pool.\
get_candidate_values(primitive_type, request_id=self._request_id, tag=default_val, quoted=quoted)
values = [
primitives.restler_custom_payload_uuid4_suffix(
current_fuzzable_value)
]
except primitives.CandidateValueException:
_raise_dict_err(primitive_type, default_val)
except Exception as err:
_handle_exception(primitive_type, default_val, err)
elif primitive_type == primitives.REFRESHABLE_AUTHENTICATION_TOKEN:
values = [primitives.restler_refreshable_authentication_token]
# Handle all the rest
else:
values = candidate_values_pool.get_fuzzable_values(
primitive_type, default_val, self._request_id, quoted)
if Settings().fuzzing_mode == 'random-walk' and not preprocessing:
random.shuffle(values)
if len(values) == 0:
_raise_dict_err(primitive_type, current_fuzzable_tag)
fuzzable.append(values)
# lazy generation of pool for candidate values
combinations_pool = itertools.product(*fuzzable)
combinations_pool = itertools.islice(combinations_pool,
Settings().max_combinations)
# skip combinations, if asked to
for _ in range(skip):
next(combinations_pool)
# for each combination's values render dynamic primitives and resolve
# dependent variables
for ind, values in enumerate(combinations_pool):
values = list(values)
values = request_utilities.resolve_dynamic_primitives(
values, candidate_values_pool)
rendered_data = "".join(values)
yield rendered_data, parser
def delete_create_once_resources(destructors, fuzzing_requests):
""" Iterates through each destructor request and sends it to the server
@param destructors: A list of destructor requests to send
@type destructors: list(Request)
@param fuzzing_requests: The global collection of requests to fuzz
@type fuzzing_requests: FuzzingRequestCollection
@return: None
@rtype : None
"""
if not destructors:
return
candidate_values_pool = GrammarRequestCollection().candidate_values_pool
logger.write_to_main("\nRendering for create-once resource destructors:\n")
for destructor in destructors:
status_codes = []
try:
logger.write_to_main(
f"{formatting.timestamp()}: Endpoint - {destructor.endpoint_no_dynamic_objects}"
)
logger.write_to_main(
f"{formatting.timestamp()}: Hex Def - {destructor.method_endpoint_hex_definition}"
)
seq = sequences.Sequence([destructor])
renderings = seq.render(
GrammarRequestCollection().candidate_values_pool,
None,
postprocessing=True)
if not renderings.valid:
logger.write_to_main(
f"{formatting.timestamp()}: Rendering INVALID")
else:
logger.write_to_main(
f"{formatting.timestamp()}: Rendering VALID")
logger.format_rendering_stats_definition(
destructor,
GrammarRequestCollection().candidate_values_pool)
if Settings().in_smoke_test_mode():
destructor.stats.request_order = 'Postprocessing'
destructor.stats.valid = 1
destructor.stats.status_code = renderings.final_request_response.status_code
destructor.stats.status_text = renderings.final_request_response.status_text
destructor.stats.sample_request.set_request_stats(
renderings.sequence.sent_request_data_list[-1].
rendered_data)
destructor.stats.sample_request.set_response_stats(
renderings.final_request_response,
renderings.final_response_datetime)
except Exception as error:
msg = f"Failed to delete create_once resource: {error!s}"
logger.raw_network_logging(msg)
logger.write_to_main(msg, print_to_console=True)
if Settings().in_smoke_test_mode():
destructor.stats.request_order = 'Postprocessing'
destructor.stats.valid = 0
if renderings and renderings.final_request_response:
destructor.stats.status_code = renderings.final_request_response.status_code
destructor.stats.status_text = renderings.final_request_response.status_text
destructor.stats.error_msg = renderings.final_request_response.body
destructor.stats.sample_request.set_request_stats(
renderings.sequence.sent_request_data_list[-1].
rendered_data)
destructor.stats.sample_request.set_response_stats(
renderings.final_request_response,
renderings.final_response_datetime)
pass
Monitor().current_fuzzing_generation += 1
logger.print_request_rendering_stats(candidate_values_pool,
fuzzing_requests, Monitor(),
fuzzing_requests.size_all_requests,
logger.POSTPROCESSING_GENERATION,
None)
def try_async_poll(request_data, response, max_async_wait_time):
""" Helper that will poll the server until a certain resource
becomes available.
This function will check the response from a PUT or POST request to
see if it contains one of the known async resource creations strings and,
if so, will poll the URL specified in the response until the resource is
said to be available or a max timeout has been reached.
@param request_data: The request that was sent's data string
@type request_data: Str
@param response: The response returned after @request was sent
@type response: Str
@param max_async_wait_time: The maximum amount of time we will wait (in
seconds) for the resource to become available
@type max_async_wait_time: Int
@return: A tuple containing:
- The response for parsing, which will either be the @response
passed to this function or the response from the GET request
if it was an async request.
- A boolean that is True if there was an error when creating the
resource
- A Boolean that tells the caller whether or not async polling was
attempted
@rtype : Tuple(HttpResponse, boolean, boolean)
"""
from utils.logger import raw_network_logging as RAW_LOGGING
from utils.logger import print_async_results as LOG_RESULTS
response_to_parse = response
resource_error = False
async_waited = False
if Settings().wait_for_async_resource_creation and max_async_wait_time > 0\
and (request_data.startswith("PUT") or request_data.startswith("POST")):
# Get the request used for polling the resource availability
data, data_in_poll_response = get_polling_request(response)
if data:
async_waited = True
poll_wait_seconds = 1
start_time = time.time()
RAW_LOGGING("Waiting for resource to be available...")
while (time.time() - start_time) < max_async_wait_time:
try:
# Send the polling request
poll_response = request_utilities.send_request_data(data)
time_str = str(round((time.time() - start_time), 2))
if data_in_poll_response:
# If this returned a '200' status code, the response should contain the parsable data.
# Otherwise, continue to poll as the resource has not yet been created. These types will
# return a '202 - Accepted' while the resource is still being created.
if poll_response.status_code == '200':
LOG_RESULTS(
request_data,
f"Resource creation succeeded after {time_str} seconds."
)
# Break and return the polling response to be parsed
response_to_parse = poll_response
break
else:
# The data is not in the polling response and must be fetched separately.
# This comes from Azure-Async responses that do not contain a Location: field
# Check for the status of the resource
response_body = json.loads(poll_response.json_body)
done = str(response_body["status"]).lower()
if done == "succeeded":
LOG_RESULTS(
request_data,
f"Resource creation succeeded after {time_str} seconds."
)
get_response = try_parse_GET_request(request_data)
if get_response:
response_to_parse = get_response
break
elif done == "failed":
LOG_RESULTS(
request_data,
f"The server reported that the resource creation Failed after {time_str} seconds."
)
resource_error = True
break
except json.JSONDecodeError as err:
LOG_RESULTS(
request_data,
"Failed to parse body of async response, retrying.")
# This may have been due to a connection failure. Retry until max_async_wait_time.
time.sleep(poll_wait_seconds)
continue
except Exception as err:
LOG_RESULTS(
request_data,
f"An exception occurred while parsing the poll message: {err!s}"
)
break
try:
if not poll_response.status_code.startswith('2'):
LOG_RESULTS(
request_data,
f"Resource creation failed after {time_str} seconds"
f" because status code '{poll_response.status_code}' was received."
)
break
except Exception as err:
LOG_RESULTS(
request_data,
f"An exception occurred while parsing the poll message: {err!s}"
)
break
time.sleep(poll_wait_seconds)
else:
RAW_LOGGING(
"Resource polling timed out before the resource was available."
)
LOG_RESULTS(
request_data,
f"Failed to create resource in {max_async_wait_time} seconds."
)
RAW_LOGGING("Attempting to get resources from GET request...")
get_response = try_parse_GET_request(request_data)
if get_response:
# Use response from the GET request for parsing. If the GET request failed,
# the caller will know to try and use the original PUT response for parsing
response_to_parse = get_response
return response_to_parse, resource_error, async_waited
def render(self,
candidate_values_pool,
lock,
preprocessing=False,
postprocessing=False):
""" Core routine that performs the rendering of restler sequences. In
principal all requests of a sequence are being constantly rendered with
a specific values combination @param request._current_combination_id
which we know in the past led to a valid rendering and only the last
request of the sequence is being rendered iteratively with all feasible
value combinations. Each time a "valid rendering" is found for the last
request of the sequence (where "valid rendering" is defined according
to "VALID_CODES"), the routine returns a new sequence which has an
end-to-end (i.e., all requests) "valid rendering" and can be added in
the sequences collection in order to be used in the future as a building
block for longer sequences.
@param candidate_values_pool: The pool of values for primitive types.
@type candidate_values_pool: Dict
@param lock: Lock object used for sync of more than one fuzzing jobs.
@type lock: thread.Lock object
@param preprocessing: Set to true if rendering during preprocessing
@type preprocessing: Bool
@return: A RenderedSequence object containing the sequence, the final
request's response, whether or not the final request received
a valid status code, and a FailureInformation enum if there was
a failure or bug detected during rendering.
@rtype : RenderedSequence
"""
# Try rendering all primitive type value combinations for last request
request = self.last_request
# for clarity reasons, don't log requests whose render iterator is over
if request._current_combination_id <\
request.num_combinations(candidate_values_pool):
CUSTOM_LOGGING(self, candidate_values_pool)
self._sent_request_data_list = []
for rendered_data, parser in\
request.render_iter(candidate_values_pool,
skip=request._current_combination_id,
preprocessing=preprocessing):
# Hold the lock (because other workers may be rendering the same
# request) and check whether the current rendering is known from the
# past to lead to invalid status codes. If so, skip the current
# rendering.
if lock is not None:
lock.acquire()
should_skip = Monitor().is_invalid_rendering(request)
if lock is not None:
lock.release()
# Skip the loop and don't forget to increase the counter.
if should_skip:
RAW_LOGGING("Skipping rendering: {}".\
format(request._current_combination_id))
request._current_combination_id += 1
continue
# Clean up internal state
self.status_codes = []
dependencies.reset_tlb()
sequence_failed = False
# Step A: Static template rendering
# Render last known valid combination of primitive type values
# for every request until the last
for i in range(len(self.requests) - 1):
prev_request = self.requests[i]
prev_rendered_data, prev_parser =\
prev_request.render_current(candidate_values_pool,
preprocessing=preprocessing)
# substitute reference placeholders with resolved values
if not Settings().ignore_dependencies:
prev_rendered_data =\
self.resolve_dependencies(prev_rendered_data)
prev_req_async_wait = Settings(
).get_max_async_resource_creation_time(prev_request.request_id)
prev_producer_timing_delay = Settings(
).get_producer_timing_delay(prev_request.request_id)
prev_response = request_utilities.send_request_data(
prev_rendered_data)
prev_response_to_parse, resource_error, async_waited = async_request_utilities.try_async_poll(
prev_rendered_data, prev_response, prev_req_async_wait)
prev_parser_threw_exception = False
# Response may not exist if there was an error sending the request or a timeout
if prev_parser and prev_response_to_parse:
prev_parser_threw_exception = not request_utilities.call_response_parser(
prev_parser, prev_response_to_parse, prev_request)
prev_status_code = prev_response.status_code
# If the async logic waited for the resource, this wait already included the required
# producer timing delay. Here, set the producer timing delay to zero, so this wait is
# skipped both below for this request and during replay
if async_waited:
prev_producer_timing_delay = 0
else:
prev_req_async_wait = 0
self.append_data_to_sent_list(prev_rendered_data, prev_parser,
prev_response,
prev_producer_timing_delay,
prev_req_async_wait)
if not prev_status_code:
logger.write_to_main(
f"Error: Failed to get status code during valid sequence re-rendering.\n"
)
sequence_failed = True
break
if prev_response.has_bug_code():
BugBuckets.Instance().update_bug_buckets(self,
prev_status_code,
reproduce=False,
lock=lock)
sequence_failed = True
break
if prev_parser_threw_exception:
logger.write_to_main(
"Error: Parser exception occurred during valid sequence re-rendering.\n"
)
sequence_failed = True
break
if resource_error:
logger.write_to_main(
"Error: The resource was left in a Failed state after creation during valid sequence re-rendering.\n"
)
sequence_failed = True
break
# If the previous request is a resource generator and we did not perform an async resource
# creation wait, then wait for the specified duration in order for the backend to have a
# chance to create the resource.
if prev_producer_timing_delay > 0 and prev_request.is_resource_generator(
):
print(
f"Pausing for {prev_producer_timing_delay} seconds, request is a generator..."
)
time.sleep(prev_producer_timing_delay)
# register latest client/server interaction
timestamp_micro = int(time.time() * 10**6)
self.status_codes.append(
status_codes_monitor.RequestExecutionStatus(
timestamp_micro,
prev_request.hex_definition, prev_status_code,
prev_response.has_valid_code(), False))
if sequence_failed:
self.status_codes.append(
status_codes_monitor.RequestExecutionStatus(
int(time.time() * 10**6), request.hex_definition,
RESTLER_INVALID_CODE, False, True))
Monitor().update_status_codes_monitor(self, self.status_codes,
lock)
return RenderedSequence(
failure_info=FailureInformation.SEQUENCE)
# Step B: Dynamic template rendering
# substitute reference placeholders with ressoved values
# for the last request
if not Settings().ignore_dependencies:
rendered_data = self.resolve_dependencies(rendered_data)
# Render candidate value combinations seeking for valid error codes
request._current_combination_id += 1
req_async_wait = Settings().get_max_async_resource_creation_time(
request.request_id)
response = request_utilities.send_request_data(rendered_data)
response_to_parse, resource_error, _ = async_request_utilities.try_async_poll(
rendered_data, response, req_async_wait)
parser_exception_occurred = False
# Response may not exist if there was an error sending the request or a timeout
if parser and response_to_parse:
parser_exception_occurred = not request_utilities.call_response_parser(
parser, response_to_parse, request)
status_code = response.status_code
if not status_code:
return RenderedSequence(None)
self.append_data_to_sent_list(rendered_data,
parser,
response,
max_async_wait_time=req_async_wait)
rendering_is_valid = not parser_exception_occurred\
and not resource_error\
and response.has_valid_code()
# register latest client/server interaction and add to the status codes list
response_datetime = datetime.datetime.now(datetime.timezone.utc)
timestamp_micro = int(response_datetime.timestamp() * 10**6)
self.status_codes.append(
status_codes_monitor.RequestExecutionStatus(
timestamp_micro, request.hex_definition, status_code,
rendering_is_valid, False))
# add sequence's error codes to bug buckets.
if response.has_bug_code():
BugBuckets.Instance().update_bug_buckets(self,
status_code,
lock=lock)
Monitor().update_status_codes_monitor(self, self.status_codes,
lock)
# Register current rendering's status.
if lock is not None:
lock.acquire()
Monitor().update_renderings_monitor(request, rendering_is_valid)
if lock is not None:
lock.release()
if Monitor().remaining_time_budget <= 0 and not postprocessing:
raise TimeOutException("Exceeded Timeout")
if lock is not None:
lock.acquire()
# Deep copying here will try copying anything the class has access
# to including the shared client monitor, which we update in the
# above code block holding the lock, but then we release the
# lock and one thread can be updating while another is copying.
# This is a typlical nasty read after write syncronization bug.
duplicate = copy.deepcopy(self)
if lock is not None:
lock.release()
datetime_format = "%Y-%m-%d %H:%M:%S"
# return a rendered clone if response indicates a valid status code
if rendering_is_valid or Settings().ignore_feedback:
return RenderedSequence(
duplicate,
valid=True,
final_request_response=response,
response_datetime=response_datetime.strftime(
datetime_format))
else:
information = None
if response.has_valid_code():
if parser_exception_occurred:
information = FailureInformation.PARSER
elif resource_error:
information = FailureInformation.RESOURCE_CREATION
elif response.has_bug_code():
information = FailureInformation.BUG
return RenderedSequence(
duplicate,
valid=False,
failure_info=information,
final_request_response=response,
response_datetime=response_datetime.strftime(
datetime_format))
return RenderedSequence(None)
