def RenderResponse(request):
"""Handles given HTTP request with one of the available API renderers."""
renderer, route_args = GetRendererForRequest(request)
if renderer.method == "GET":
if hasattr(request.GET, "dict"):
request_data_obj = utils.DataObject(request.GET.dict())
else:
request_data_obj = utils.DataObject(request.GET)
elif renderer.method == "POST":
if hasattr(request.POST, "dict"):
request_data_obj = utils.DataObject(request.POST.dict())
else:
request_data_obj = utils.DataObject(request.POST)
for k, v in route_args.items():
request_data_obj[k] = v
request_data_obj.token = BuildToken(request, renderer.max_execution_time)
rendered_data = renderer.Render(request_data_obj)
response = http.HttpResponse(content_type="application/json")
response.write(json.dumps(rendered_data))
return response
def Check(self, method, url, payload=None, replace=None):
"""Records output of a given url accessed with a given method.
Args:
method: HTTP method. May be "GET" or "POST".
url: String repesenting an url.
payload: JSON-able payload that will be sent when "POST" method is used.
replace: Dictionary of key->value pairs. In the recorded JSON output
every "key" string will be replaced with its "value"
counterpart. This way we can properly handle dynamically
generated values (like hunts IDs) in the regression data.
Raises:
ValueError: if unsupported method argument is passed. Currently only
"GET" is supported.
"""
parsed_url = urlparse.urlparse(url)
request = utils.DataObject(method=method,
scheme="http",
path=parsed_url.path,
environ={
"SERVER_NAME": "foo.bar",
"SERVER_PORT": 1234
},
user="******")
request.META = {"CONTENT_TYPE": "application/json"}
if method == "GET":
request.GET = dict(urlparse.parse_qsl(parsed_url.query))
elif method == "POST":
request.body = json.dumps(payload)
else:
raise ValueError("Unsupported method: %s." % method)
with self.NoAuthorizationChecks():
http_response = http_api.RenderHttpResponse(request)
content = http_response.content
xssi_token = ")]}'\n"
if content.startswith(xssi_token):
content = content[len(xssi_token):]
if replace:
if hasattr(replace, "__call__"):
replace = replace()
for substr, repl in replace.items():
content = content.replace(substr, repl)
url = url.replace(substr, repl)
parsed_content = json.loads(content)
self.checks.append(
dict(method=method,
url=url,
test_class=self.__class__.__name__,
response=parsed_content))
def testRendersListOfHuntObjects(self):
for i in range(10):
self.CreateSampleHunt("hunt_%d" % i)
result = self.renderer.Render(utils.DataObject(token=self.token))
descriptions = set(r["description"] for r in result)
self.assertEqual(len(descriptions), 10)
for i in range(10):
self.assertTrue("hunt_%d" % i in descriptions)
def GetConfigMockClass(sections=None):
"""Mocks a configuration file for use by the API handler.
Args:
sections: A dict containing one key per config section
with a value of a dict containing one key per config parameter name
and a value of config parameter value. (default {})
Returns:
A class to be used as a config mock.
"""
if sections is None:
sections = {}
missing = object()
type_infos = []
values = {}
raw_values = {}
default_values = {}
for section_name, section in sections.iteritems():
for parameter_name, parameter_data in section.iteritems():
name = "%s.%s" % (section_name, parameter_name)
descriptor = utils.DataObject(section=section_name, name=name)
type_infos.append(descriptor)
if "value" in parameter_data:
values[name] = parameter_data["value"]
if "raw_value" in parameter_data:
raw_values[name] = parameter_data["raw_value"]
if "default_value" in parameter_data:
default_values[name] = parameter_data["default_value"]
def Get(parameter, default=missing):
try:
return values[parameter]
except KeyError:
if default is missing:
return default_values[parameter]
return default
def GetRaw(parameter, default=missing):
try:
return raw_values[parameter]
except KeyError:
if default is missing:
return default_values[parameter]
return default
return {"Get": Get, "GetRaw": GetRaw, "type_infos": type_infos}
def testHuntListIsSortedInReversedCreationTimestampOrder(self):
for i in range(10):
with test_lib.FakeTime(i * 1000):
self.CreateSampleHunt("hunt_%d" % i)
result = self.renderer.Render(utils.DataObject(token=self.token))
create_times = [r["create_time"] for r in result]
self.assertEqual(len(create_times), 10)
for index, expected_time in enumerate(reversed(range(10))):
self.assertEqual(create_times[index], expected_time * 1000000000)
def testRendersSubrangeOfListOfHuntObjects(self):
for i in range(10):
with test_lib.FakeTime(i * 1000):
self.CreateSampleHunt("hunt_%d" % i)
result = self.renderer.Render(
utils.DataObject(offset=2, count=2, token=self.token))
create_times = [r["create_time"] for r in result]
self.assertEqual(len(create_times), 2)
self.assertEqual(create_times[0], 7 * 1000000000)
self.assertEqual(create_times[1], 6 * 1000000000)
def _CreateRequest(self, method, path, query_parameters=None):
request = utils.DataObject()
request.method = method
request.path = path
request.scheme = "http"
request.environ = {"SERVER_NAME": "foo.bar", "SERVER_PORT": 1234}
request.user = "******"
request.args = query_parameters or {}
request.headers = {}
request.get_data = lambda as_text=False: ""
return request
def InitializeContext(self, args):
"""Initializes the context of this flow."""
if args is None:
args = rdfvalue.FlowRunnerArgs()
output_collection = self._CreateOutputCollection(args)
# Output collection is nullified when flow is terminated, so we're
# keeping the urn separately for further reference.
output_urn = (output_collection is not None) and output_collection.urn
output_plugins_states = []
for plugin_descriptor in args.output_plugins:
plugin_class = plugin_descriptor.GetPluginClass()
plugin = plugin_class(output_urn,
args=plugin_descriptor.plugin_args,
token=self.token)
try:
plugin.Initialize()
output_plugins_states.append((plugin_descriptor, plugin.state))
except Exception as e: # pylint: disable=broad-except
self.Log("Plugin %s failed to initialize (%s), ignoring it." %
(plugin, e))
context = utils.DataObject(
args=args,
backtrace=None,
client_resources=rdfvalue.ClientResources(),
create_time=rdfvalue.RDFDatetime().Now(),
creator=args.creator or self.token.username,
current_state="Start",
# If not None, kill-stuck-flow notification is scheduled at the given
# time.
kill_timestamp=None,
network_bytes_sent=0,
next_outbound_id=1,
next_processed_request=1,
next_states=set(),
output=output_collection,
output_plugins_states=output_plugins_states,
output_urn=output_urn,
outstanding_requests=0,
remaining_cpu_quota=args.cpu_limit,
state=rdfvalue.Flow.State.RUNNING,
# Have we sent a notification to the user.
user_notified=False,
)
# Store the context in the flow_obj for next time.
self.flow_obj.state.Register("context", context)
return context
def testRendersAff4ObjectWithGivenPath(self):
with test_lib.FakeTime(42):
with aff4.FACTORY.Create("aff4:/tmp/foo/bar",
"AFF4Volume",
token=self.token) as _:
pass
result = self.renderer.Render(
utils.DataObject(aff4_path="tmp/foo/bar", token=self.token))
self.assertEqual(result["urn"], "aff4:/tmp/foo/bar")
self.assertEqual(result["aff4_class"], "AFF4Volume")
self.assertEqual(result["age_policy"], "NEWEST_TIME")
self.assertEqual(result["attributes"]["metadata:last"], 42000000)
def _CreateRequest(self, method, path, query_parameters=None):
if not query_parameters:
query_parameters = {}
request = utils.DataObject()
request.method = method
request.path = path
request.scheme = "http"
request.environ = {"SERVER_NAME": "foo.bar", "SERVER_PORT": 1234}
request.user = "******"
if method == "GET":
request.GET = query_parameters
request.META = {}
return request
def _CreateRequest(self,
method,
path,
username="******",
query_parameters=None):
request = utils.DataObject()
request.method = method
request.path = path
request.scheme = "http"
request.environ = {"SERVER_NAME": "foo.bar", "SERVER_PORT": 1234}
request.user = username
request.args = query_parameters or {}
request.content_type = "application/json; charset=utf-8"
request.headers = {}
request.get_data = lambda as_text=False: ""
return request
def _CreateRequest(self, method, path, username="******",
query_parameters=None):
if not query_parameters:
query_parameters = {}
request = utils.DataObject()
request.method = method
request.path = path
request.scheme = "http"
request.environ = {"SERVER_NAME": "foo.bar", "SERVER_PORT": 1234}
request.user = username
if method in ["GET", "HEAD"]:
request.GET = query_parameters
request.META = {"CONTENT_TYPE": "application/json; charset=utf-8"}
request.body = ""
return request
def testFlowState(self):
state = rdf_flows.FlowState()
state.Register("teststate", 1)
state.teststate = 100
state.Register("context", utils.DataObject())
state.context.testcontext = 50
s = state.SerializeToString()
new_state = rdf_flows.FlowState()
new_state.ParseFromString(s)
self.assertEqual(new_state.teststate, 100)
self.assertEqual(new_state.context.testcontext, 50)
# context and teststate
self.assertEqual(len(new_state), 2)
self.assertEqual(len(new_state.context), 1)
def testReturnsChildrenListWithTimestamps(self):
with test_lib.FakeTime(42):
with aff4.FACTORY.Create("aff4:/tmp/foo/bar1",
"AFF4Volume",
token=self.token) as _:
pass
with test_lib.FakeTime(43):
with aff4.FACTORY.Create("aff4:/tmp/foo/bar2",
"AFF4Volume",
token=self.token) as _:
pass
result = self.renderer.Render(
utils.DataObject(aff4_path="tmp/foo", token=self.token))
result = sorted(result, key=lambda x: x[0])
self.assertEqual(result, [["aff4:/tmp/foo/bar1", 42000000],
["aff4:/tmp/foo/bar2", 43000000]])
def InitializeContext(self, args):
"""Initializes the context of this hunt."""
if args is None:
args = HuntRunnerArgs()
# For large hunts, checking client limits creates a high load on the foreman
# since it needs to read the hunt object's client list. We therefore don't
# allow setting it for large hunts. Note that client_limit of 0 means
# unlimited which is allowed (the foreman then does not need to check the
# client list)..
if args.client_limit > 1000:
raise RuntimeError("Please specify client_limit <= 1000.")
context = utils.DataObject(
args=args,
backtrace=None,
client_resources=rdfvalue.ClientResources(),
create_time=rdfvalue.RDFDatetime().Now(),
creator=self.token.username,
expires=rdfvalue.RDFDatetime().Now(),
# If not None, kill-stuck-flow notification is scheduled at the given
# time.
kill_timestamp=None,
network_bytes_sent=0,
next_client_due=0,
next_outbound_id=1,
next_processed_request=1,
next_states=set(),
outstanding_requests=0,
current_state=None,
start_time=rdfvalue.RDFDatetime().Now(),
# Hunts are always in the running state.
state=rdfvalue.Flow.State.RUNNING,
usage_stats=rdfvalue.ClientResourcesStats(),
remaining_cpu_quota=args.cpu_limit,
)
# Store the context in the flow_obj for next time.
self.flow_obj.state.Register("context", context)
return context
def InitializeContext(self, args):
"""Initializes the context of this flow."""
if args is None:
args = rdfvalue.FlowRunnerArgs()
output_collection = self._CreateOutputCollection(args)
context = utils.DataObject(
args=args,
backtrace=None,
client_resources=rdfvalue.ClientResources(),
create_time=rdfvalue.RDFDatetime().Now(),
creator=args.creator or self.token.username,
current_state="Start",
# If not None, kill-stuck-flow notification is scheduled at the given
# time.
kill_timestamp=None,
network_bytes_sent=0,
next_outbound_id=1,
next_processed_request=1,
next_states=set(),
output=output_collection,
# Output collection is nullified when flow is terminated, so we're
# keeping the urn separately for further reference.
output_urn=(output_collection is not None)
and output_collection.urn,
outstanding_requests=0,
remaining_cpu_quota=args.cpu_limit,
state=rdfvalue.Flow.State.RUNNING,
# Have we sent a notification to the user.
user_notified=False,
)
# Store the context in the flow_obj for next time.
self.flow_obj.state.Register("context", context)
return context
def _HandleConfig(self, sections):
with self._ConfigStub(sections):
mock_request = utils.DataObject()
result = self.handler.Handle(mock_request)
return result
def __init__(self, initializer=None, age=None):
self.data = utils.DataObject()
super(FlowState, self).__init__(initializer=initializer, age=age)
def Check(self, method, url, payload=None, replace=None):
"""Records output of a given url accessed with a given method.
Args:
method: HTTP method. May be "GET" or "POST".
url: String repesenting an url.
payload: JSON-able payload that will be sent when "POST" method is used.
replace: Dictionary of key->value pairs. In the recorded JSON output
every "key" string will be replaced with its "value"
counterpart. This way we can properly handle dynamically
generated values (like hunts IDs) in the regression data.
Raises:
ValueError: if unsupported method argument is passed. Currently only
"GET", "POST", "DELETE" and "PATCH" are supported.
RuntimeError: if request was handled by an unexpected API method (
every test is annotated with an "api_method" attribute
that points to the expected API method).
"""
if self.use_api_v2:
url = url.replace("/api/", "/api/v2/")
parsed_url = urlparse.urlparse(url)
request = utils.DataObject(method=method,
scheme="http",
path=parsed_url.path,
environ={
"SERVER_NAME": "foo.bar",
"SERVER_PORT": 1234
},
user=self.token.username,
body="")
request.META = {"CONTENT_TYPE": "application/json"}
if method == "GET":
request.GET = dict(urlparse.parse_qsl(parsed_url.query))
elif method in ["POST", "DELETE", "PATCH"]:
# NOTE: this is a temporary trick. Payloads in regression tests
# are using the API v1 (non-proto3) format. Here we're reparsing
# them and serializing as proto3 JSON.
# TODO(user): Make regression tests payload format-agnostic.
# I.e. use protobuf and API client library to send requests.
if self.use_api_v2 and payload:
router_matcher = http_api.RouterMatcher()
_, metadata, _ = router_matcher.MatchRouter(request)
rdf_args = metadata.args_type()
rdf_args.FromDict(payload)
proto_args = metadata.args_type.protobuf()
proto_args.ParseFromString(rdf_args.SerializeToString())
request.body = json_format.MessageToJson(proto_args)
payload = json.loads(request.body)
else:
request.body = json.dumps(payload or "")
else:
raise ValueError("Unsupported method: %s." % method)
with self.NoAuthorizationChecks():
http_response = http_api.RenderHttpResponse(request)
api_method = http_response["X-API-Method"]
if api_method != self.__class__.api_method:
raise RuntimeError("Request was handled by an unexpected method. "
"Expected %s, got %s." %
(self.__class__.api_method, api_method))
if hasattr(http_response, "streaming_content"):
# We don't know the nature of response content, but we force it to be
# unicode. It's a strategy that's good enough for testing purposes.
content = utils.SmartUnicode("".join(
http_response.streaming_content))
else:
content = http_response.content
xssi_token = ")]}'\n"
if content.startswith(xssi_token):
content = content[len(xssi_token):]
# replace the values of all tracebacks by <traceback content>
regex = re.compile(r'"traceBack": "Traceback[^"\\]*(?:\\.[^"\\]*)*"',
re.DOTALL)
content = regex.sub('"traceBack": "<traceback content>"', content)
if replace:
if hasattr(replace, "__call__"):
replace = replace()
# We reverse sort replacements by length to avoid cases when
# replacements include each other and therefore order
# of replacements affects the result.
for substr in sorted(replace, key=len, reverse=True):
repl = replace[substr]
if hasattr(substr, "sub"): # regex
content = substr.sub(repl, content)
url = substr.sub(repl, url)
else:
content = content.replace(substr, repl)
url = url.replace(substr, repl)
# We treat streaming content purely as strings and don't expect it to
# contain JSON data.
if hasattr(http_response, "streaming_content"):
parsed_content = content
else:
parsed_content = json.loads(content)
check_result = dict(api_method=api_method,
method=method,
url=url,
test_class=self.__class__.__name__,
response=parsed_content)
if payload:
check_result["request_payload"] = payload
# Type stripping only makes sense for version 1 of the API.
if not self.use_api_v2:
stripped_content = api_value_renderers.StripTypeInfo(
parsed_content)
if parsed_content != stripped_content:
check_result["type_stripped_response"] = stripped_content
self.checks.append(check_result)
def Check(self, method, url, payload=None, replace=None):
"""Records output of a given url accessed with a given method.
Args:
method: HTTP method. May be "GET" or "POST".
url: String repesenting an url.
payload: JSON-able payload that will be sent when "POST" method is used.
replace: Dictionary of key->value pairs. In the recorded JSON output
every "key" string will be replaced with its "value"
counterpart. This way we can properly handle dynamically
generated values (like hunts IDs) in the regression data.
Raises:
ValueError: if unsupported method argument is passed. Currently only
"GET", "POST", "DELETE" and "PATCH" are supported.
"""
parsed_url = urlparse.urlparse(url)
request = utils.DataObject(method=method,
scheme="http",
path=parsed_url.path,
environ={
"SERVER_NAME": "foo.bar",
"SERVER_PORT": 1234
},
user="******")
request.META = {"CONTENT_TYPE": "application/json"}
if method == "GET":
request.GET = dict(urlparse.parse_qsl(parsed_url.query))
elif method in ["POST", "DELETE", "PATCH"]:
request.body = json.dumps(payload or "")
else:
raise ValueError("Unsupported method: %s." % method)
with self.NoAuthorizationChecks():
http_response = http_api.RenderHttpResponse(request)
content = http_response.content
xssi_token = ")]}'\n"
if content.startswith(xssi_token):
content = content[len(xssi_token):]
# replace the values of all tracebacks by <traceback content>
regex = re.compile(r'"traceBack": "Traceback[^"\\]*(?:\\.[^"\\]*)*"',
re.DOTALL)
content = regex.sub('"traceBack": "<traceback content>"', content)
if replace:
if hasattr(replace, "__call__"):
replace = replace()
for substr, repl in replace.items():
if hasattr(substr, "sub"): # regex
content = substr.sub(repl, content)
url = substr.sub(repl, url)
else:
content = content.replace(substr, repl)
url = url.replace(substr, repl)
parsed_content = json.loads(content)
check_result = dict(method=method,
url=url,
test_class=self.__class__.__name__,
response=parsed_content)
if payload:
check_result["request_payload"] = payload
stripped_content = http_api.HttpRequestHandler.StripTypeInfo(
parsed_content)
if parsed_content != stripped_content:
check_result["type_stripped_response"] = stripped_content
self.checks.append(check_result)
def _RenderConfig(self, sections):
with self._ConfigStub(sections):
mock_request = utils.DataObject()
rendering = self.renderer.Render(mock_request)
return rendering
