def GetFlowResultsByTag(client_id, flow_id):
utils.AssertType(client_id, unicode)
utils.AssertType(flow_id, unicode)
results = data_store.REL_DB.ReadFlowResults(client_id, flow_id, 0,
sys.maxsize)
return {r.tag or "": r.payload for r in results}
def ApplyParserToResponses(processor_obj, responses, flow_obj):
"""Parse responses using the specified processor and the right args.
Args:
processor_obj: A Processor object that inherits from Parser.
responses: A list of, or single response depending on the processors
process_together setting.
flow_obj: An artifact collection flow.
Raises:
RuntimeError: On bad parser.
Returns:
An iterator of the processor responses.
"""
# TODO(hanuszczak): Is it ever possible that we have `None`?
if not processor_obj:
# We don't do any parsing, the results are raw as they came back.
# If this is an RDFValue we don't want to unpack it further
if isinstance(responses, rdfvalue.RDFValue):
result_iterator = [responses]
else:
result_iterator = responses
else:
# We have some processors to run.
state = flow_obj.state
# TODO(hanuszczak): Implement new-style parsing method with support for
# multiple responses.
if processor_obj.process_together:
# We are processing things in a group which requires specialized
# handling by the parser. This is used when multiple responses need to
# be combined to parse successfully. E.g parsing passwd and shadow files
# together.
parse_method = processor_obj.ParseMultiple
if isinstance(processor_obj, parser.FileParser):
# TODO(amoser): This is very brittle, one day we should come
# up with a better API here.
urns = [r.AFF4Path(flow_obj.client_urn) for r in responses]
file_objects = list(
aff4.FACTORY.MultiOpen(urns, token=flow_obj.token))
file_objects.sort(key=lambda file_object: file_object.urn)
stats = sorted(responses, key=lambda r: r.pathspec.path)
result_iterator = parse_method(stats, file_objects,
state.knowledge_base)
elif isinstance(processor_obj, parser.RegistryParser):
result_iterator = parse_method(responses, state.knowledge_base)
elif isinstance(processor_obj, parser.ArtifactFilesParser):
result_iterator = parse_method(responses, state.knowledge_base,
flow_obj.args.path_type)
else:
raise RuntimeError("Unsupported parser detected %s" %
processor_obj)
else:
utils.AssertType(processor_obj, parser.SingleResponseParser)
result_iterator = processor_obj.ParseResponse(
state.knowledge_base, responses, flow_obj.args.path_type)
return result_iterator
def ExpandGlobs(path, opts=None):
"""Performs glob expansion on a given path.
Path can contain regular glob elements (such as `**`, `*`, `?`, `[a-z]`). For
example, having files `foo`, `bar`, `baz` glob expansion of `ba?` will yield
`bar` and `baz`.
Args:
path: A path to expand.
opts: A `PathOpts` object.
Returns:
Generator over all possible glob expansions of a given path.
Raises:
ValueError: If given path is empty or relative.
"""
utils.AssertType(path, unicode)
if not path:
raise ValueError("Path is empty")
drive, tail = os.path.splitdrive(path)
if ((platform.system() == "Windows" and not drive)
or not (tail and tail[0] == os.path.sep)):
raise ValueError("Path '%s' is not absolute" % path)
root_dir = os.path.join(drive, os.path.sep).upper()
components = list(ParsePath(tail[1:], opts=opts))
return _ExpandComponents(root_dir, components)
def _CheckAccess(self, username, subject_id, approval_type):
"""Checks access to a given subject by a given user."""
utils.AssertType(subject_id, unicode)
cache_key = (username, subject_id, approval_type)
try:
self.acl_cache.Get(cache_key)
stats.STATS.IncrementCounter("approval_searches", fields=["-", "cache"])
return True
except KeyError:
stats.STATS.IncrementCounter("approval_searches", fields=["-", "reldb"])
approvals = data_store.REL_DB.ReadApprovalRequests(
username, approval_type, subject_id=subject_id, include_expired=False)
errors = []
for approval in approvals:
try:
approval_checks.CheckApprovalRequest(approval)
self.acl_cache.Put(cache_key, True)
return
except access_control.UnauthorizedAccess as e:
errors.append(e)
subject = approval_checks.BuildLegacySubject(subject_id, approval_type)
if not errors:
raise access_control.UnauthorizedAccess(
"No approval found.", subject=subject)
else:
raise access_control.UnauthorizedAccess(
" ".join(utils.SmartStr(e) for e in errors), subject=subject)
def ParseFromUnicode(self, initializer):
utils.AssertType(initializer, unicode)
# Strip off the aff4: prefix if necessary.
if initializer.startswith("aff4:/"):
initializer = initializer[5:]
self._string_urn = utils.NormalizePath(initializer)
def ExpandGroups(path):
"""Performs group expansion on a given path.
For example, given path `foo/{bar,baz}/{quux,norf}` this method will yield
`foo/bar/quux`, `foo/bar/norf`, `foo/baz/quux`, `foo/baz/norf`.
Args:
path: A path to expand.
Yields:
Paths that can be obtained from given path by expanding groups.
"""
utils.AssertType(path, unicode)
chunks = []
offset = 0
for match in PATH_GROUP_REGEX.finditer(path):
chunks.append([path[offset:match.start()]])
chunks.append(match.group("alts").split(","))
offset = match.end()
chunks.append([path[offset:]])
for prod in itertools.product(*chunks):
yield "".join(prod)
def _BuildStreamingResponse(self, binary_stream, method_name=None):
"""Builds HTTPResponse object for streaming."""
utils.AssertType(method_name, unicode)
# We get a first chunk of the output stream. This way the likelihood
# of catching an exception that may happen during response generation
# is much higher.
content = binary_stream.GenerateContent()
try:
peek = content.next()
stream = itertools.chain([peek], content)
except StopIteration:
stream = []
response = werkzeug_wrappers.Response(
response=stream,
content_type="binary/octet-stream",
direct_passthrough=True)
response.headers["Content-Disposition"] = ((
"attachment; filename=%s" % binary_stream.filename).encode("utf-8"))
if method_name:
response.headers["X-API-Method"] = method_name.encode("utf-8")
if binary_stream.content_length:
response.content_length = binary_stream.content_length
return response
def ParseFromString(self, pem_string):
utils.AssertType(pem_string, bytes)
try:
self._value = serialization.load_pem_public_key(
pem_string, backend=openssl.backend)
except (TypeError, ValueError, exceptions.UnsupportedAlgorithm) as e:
raise type_info.TypeValueError("Public key invalid: %s" % e)
def ParseResponse(self, knowledge_base, response, path_type):
del path_type # Unused.
# TODO(hanuszczak): Why some of the registry value parsers anticipate string
# response? This is stupid.
utils.AssertType(response, (rdf_client_fs.StatEntry, rdfvalue.RDFString))
return self.Parse(response, knowledge_base)
def ParseFromDatastore(self, value):
utils.AssertType(value, unicode)
# TODO(hanuszczak): We should just assign the `self._string_urn` here
# instead of including all of the parsing magic since the data store values
# should be normalized already. But sadly this is not the case and for now
# we have to deal with unnormalized values as well.
self.ParseFromString(value)
def __init__(self, data=""): utils.AssertType(data, unicode) # Set the lexer up to process a new data feed. self.Reset() # Populate internal token list with class tokens, if defined. self._tokens = self.tokens[:] # Populate the lexer with any data we got. self.buffer = data
def ParseFromString(self, initializer):
"""Create RDFRUN from string.
Args:
initializer: url string
"""
utils.AssertType(initializer, bytes)
self.ParseFromUnicode(initializer.decode("utf-8"))
def ParseEntries(self, data):
utils.AssertType(data, unicode)
self.entries = {}
self.Reset()
self.Feed(data)
self.Close()
found = set(self.entries)
if self.required.issubset(found):
return self.entries
def __init__(self, fopen=None):
"""Initializes the file parser object.
Args:
fopen: A function that returns file-like object for given pathspec.
"""
# TODO(hanuszczak): Define clear interface for file-like objects.
if fopen is not None:
utils.AssertType(fopen, types.FunctionType)
self._fopen = fopen
def ParseFromHumanReadable(self, string):
utils.AssertType(string, unicode)
upper_string = string.upper()
if upper_string == u"TRUE" or string == u"1":
self._value = 1
elif upper_string == u"FALSE" or string == u"0":
self._value = 0
else:
raise ValueError("Unparsable boolean string: `%s`" % string)
def GenerateCSRFToken(user_id, time):
"""Generates a CSRF token based on a secret key, id and time."""
utils.AssertType(user_id, unicode)
if time is not None:
utils.AssertType(time, int)
time = time or rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch()
secret = config.CONFIG.Get("AdminUI.csrf_secret_key", None)
# TODO(amoser): Django is deprecated. Remove this at some point.
if not secret:
secret = config.CONFIG["AdminUI.django_secret_key"]
digester = hmac.new(secret.encode("ascii"), digestmod=hashlib.sha256)
digester.update(user_id.encode("ascii"))
digester.update(CSRF_DELIMITER)
digester.update(unicode(time).encode("ascii"))
digest = digester.digest()
token = base64.urlsafe_b64encode(b"%s%s%d" %
(digest, CSRF_DELIMITER, time))
return token.rstrip(b"=")
def ParseResponse(self, knowledge_base, response, path_type):
del path_type # Unused.
utils.AssertType(response, rdf_client_action.ExecuteResponse)
return self.Parse(
cmd=response.request.cmd,
args=response.request.args,
stdout=response.stdout,
stderr=response.stderr,
return_val=response.exit_status,
time_taken=response.time_used,
knowledge_base=knowledge_base)
def Sign(self, message, use_pss=False):
"""Sign a given message."""
utils.AssertType(message, bytes)
# TODO(amoser): This should use PSS by default at some point.
if not use_pss:
padding_algorithm = padding.PKCS1v15()
else:
padding_algorithm = padding.PSS(
mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH)
return self._value.sign(message, padding_algorithm, hashes.SHA256())
def ParseResponse(self, knowledge_base, response, path_type):
del path_type # Unused.
utils.AssertType(response, rdf_client_fs.StatEntry)
# TODO(hanuszczak): This is a temporary hack to avoid rewriting hundreds of
# tests for file parser. Once the tests are adapted to use the new API, the
# constructor should be disallowed to take `None` as file opener as soon as
# possible.
if self._fopen is None:
raise AssertionError("Parser constructed without file opening function")
with self._fopen(response.pathspec) as filedesc:
return self.Parse(response, filedesc, knowledge_base)
def ExpandPath(path, opts=None):
"""Applies all expansion mechanisms to the given path.
Args:
path: A path to expand.
opts: A `PathOpts` object.
Yields:
All paths possible to obtain from a given path by performing expansions.
"""
utils.AssertType(path, unicode)
for grouped_path in ExpandGroups(path):
for globbed_path in ExpandGlobs(grouped_path, opts=opts):
yield globbed_path
def __init__(self, client_id, client_mock, token=None):
if not isinstance(client_id, rdf_client.ClientURN):
client_id = rdf_client.ClientURN(client_id)
if client_mock is None:
client_mock = action_mocks.InvalidActionMock()
else:
utils.AssertType(client_mock, action_mocks.ActionMock)
self._mock_task_queue = getattr(client_mock, "mock_task_queue", [])
self.client_id = client_id
self.client_mock = client_mock
self.token = token
# Well known flows are run on the front end.
self.well_known_flows = flow.WellKnownFlow.GetAllWellKnownFlows(token=token)
def __init__(self,
name,
doc=None,
args_type=None,
result_type=None,
category=None,
http_methods=None,
no_audit_log_required=False):
utils.AssertType(name, unicode)
self.name = name
self.doc = doc
self.args_type = args_type
self.result_type = result_type
self.category = category
self.http_methods = http_methods or set()
self.no_audit_log_required = no_audit_log_required
def _RunClientAction(self, action, request, parsers, path_type):
"""Runs the client action with the request and parses the result."""
responses = list(action(request))
if not parsers:
return responses
# filter parsers by process_together setting
multi_parsers = []
single_parsers = []
for parser in parsers:
# TODO(hanuszczak): This is absolutely disgusting and should be refactored
# with some kind of factory, that produces different parser instances
# depending whether we are on the server or on the client. Doing so would
# probably solve issues with this very artificial metclass registry that
# we have here.
if issubclass(parser, parser_lib.FileParser):
parser_obj = parser(vfs.VFSOpen)
else:
parser_obj = parser()
if parser_obj.process_together:
multi_parsers.append(parser_obj)
else:
single_parsers.append(parser_obj)
# parse the responses
parsed_responses = []
for response in responses:
for parser in single_parsers:
utils.AssertType(parser, parser_lib.SingleResponseParser)
parsed_responses.extend(
parser.ParseResponse(self.knowledge_base, response, path_type))
for parser in multi_parsers:
for res in ParseMultipleResponses(parser, responses, self.knowledge_base,
path_type):
parsed_responses.append(res)
return parsed_responses
def __init__(self, filename, content_generator=None, content_length=None):
"""ApiBinaryStream constructor.
Args:
filename: A file name to be used by the browser when user downloads the
file.
content_generator: A generator that yields byte chunks (of any size) to
be streamed to the user.
content_length: The length of the stream, if known upfront.
Raises:
ValueError: if content_generator is None.
"""
utils.AssertType(filename, unicode)
self.filename = filename
self.content_length = content_length
if content_generator is None:
raise ValueError("content_generator can't be None")
self.content_generator = content_generator
def _AddToHostData(self, host_data, artifact, data, parser):
"""Parse raw data collected for an artifact into the host_data table."""
utils.AssertType(data, dict)
rdfs = []
stats = []
files = []
for path, lines in iteritems(data):
stat = self.CreateStat(path)
stats.append(stat)
file_obj = io.StringIO(lines)
files.append(file_obj)
if not parser.process_together:
rdfs.extend(list(parser.Parse(stat, file_obj, None)))
if parser.process_together:
rdfs = list(parser.ParseMultiple(stats, files, None))
host_data[artifact] = self.SetArtifactData(
anomaly=[a for a in rdfs if isinstance(a, rdf_anomaly.Anomaly)],
parsed=[r for r in rdfs if not isinstance(r, rdf_anomaly.Anomaly)],
raw=stats,
results=host_data.get(artifact))
return host_data
def ParseFromString(self, pem_string):
utils.AssertType(pem_string, bytes)
try:
self._value = serialization.load_pem_private_key(
pem_string, password=None, backend=openssl.backend)
return
except (TypeError, ValueError, exceptions.UnsupportedAlgorithm) as e:
if "private key is encrypted" not in str(e):
raise type_info.TypeValueError("Private key invalid: %s" % e)
# pylint: disable=g-explicit-bool-comparison, g-equals-none
# The private key is passphrase protected, we need to see if we are
# allowed to ask the user.
#
# If allow_prompt is False, we are explicitly told that we are not.
if self.allow_prompt == False:
raise type_info.TypeValueError("Private key invalid: %s" % e)
# allow_prompt was not set, we use the context we are in to see if it
# makes sense to ask.
elif self.allow_prompt == None:
# TODO(user): dependency loop with
# core/grr_response_core/grr/config/client.py.
# pylint: disable=protected-access
if "Commandline Context" not in config_lib._CONFIG.context:
raise type_info.TypeValueError("Private key invalid: %s" % e)
# pylint: enable=protected-access
# pylint: enable=g-explicit-bool-comparison, g-equals-none
try:
# The private key is encrypted and we can ask the user for the passphrase.
password = utils.PassphraseCallback()
self._value = serialization.load_pem_private_key(
pem_string, password=password, backend=openssl.backend)
except (TypeError, ValueError, exceptions.UnsupportedAlgorithm) as e:
raise type_info.TypeValueError("Unable to load private key: %s" % e)
def ParsePath(path, opts=None):
"""Parses given path into a stream of `PathComponent` instances.
Args:
path: A path to be parsed.
opts: An `PathOpts` object.
Yields:
`PathComponent` instances corresponding to the components of the given path.
Raises:
ValueError: If path contains more than one recursive component.
"""
utils.AssertType(path, unicode)
rcount = 0
for item in path.split(os.path.sep):
component = ParsePathItem(item, opts=opts)
if isinstance(component, RecursiveComponent):
rcount += 1
if rcount > 1:
raise ValueError(
"path cannot have more than one recursive component")
yield component
def __init__(self, collection_id):
utils.AssertType(collection_id, rdfvalue.RDFURN)
super(SequentialCollection, self).__init__()
# The collection_id for this collection is a RDFURN for now.
self.collection_id = collection_id
def ParseFromHumanReadable(self, string):
utils.AssertType(string, unicode)
self._value = int(string)
def ParseFromDatastore(self, value):
utils.AssertType(value, int)
self._value = value
