def GetFlowResultsByTag(client_id, flow_id): utils.AssertType(client_id, unicode) utils.AssertType(flow_id, unicode) results = data_store.REL_DB.ReadFlowResults(client_id, flow_id, 0, sys.maxsize) return {r.tag or "": r.payload for r in results}
def ApplyParserToResponses(processor_obj, responses, flow_obj): """Parse responses using the specified processor and the right args. Args: processor_obj: A Processor object that inherits from Parser. responses: A list of, or single response depending on the processors process_together setting. flow_obj: An artifact collection flow. Raises: RuntimeError: On bad parser. Returns: An iterator of the processor responses. """ # TODO(hanuszczak): Is it ever possible that we have `None`? if not processor_obj: # We don't do any parsing, the results are raw as they came back. # If this is an RDFValue we don't want to unpack it further if isinstance(responses, rdfvalue.RDFValue): result_iterator = [responses] else: result_iterator = responses else: # We have some processors to run. state = flow_obj.state # TODO(hanuszczak): Implement new-style parsing method with support for # multiple responses. if processor_obj.process_together: # We are processing things in a group which requires specialized # handling by the parser. This is used when multiple responses need to # be combined to parse successfully. E.g parsing passwd and shadow files # together. parse_method = processor_obj.ParseMultiple if isinstance(processor_obj, parser.FileParser): # TODO(amoser): This is very brittle, one day we should come # up with a better API here. urns = [r.AFF4Path(flow_obj.client_urn) for r in responses] file_objects = list( aff4.FACTORY.MultiOpen(urns, token=flow_obj.token)) file_objects.sort(key=lambda file_object: file_object.urn) stats = sorted(responses, key=lambda r: r.pathspec.path) result_iterator = parse_method(stats, file_objects, state.knowledge_base) elif isinstance(processor_obj, parser.RegistryParser): result_iterator = parse_method(responses, state.knowledge_base) elif isinstance(processor_obj, parser.ArtifactFilesParser): result_iterator = parse_method(responses, state.knowledge_base, flow_obj.args.path_type) else: raise RuntimeError("Unsupported parser detected %s" % processor_obj) else: utils.AssertType(processor_obj, parser.SingleResponseParser) result_iterator = processor_obj.ParseResponse( state.knowledge_base, responses, flow_obj.args.path_type) return result_iterator
def ExpandGlobs(path, opts=None): """Performs glob expansion on a given path. Path can contain regular glob elements (such as `**`, `*`, `?`, `[a-z]`). For example, having files `foo`, `bar`, `baz` glob expansion of `ba?` will yield `bar` and `baz`. Args: path: A path to expand. opts: A `PathOpts` object. Returns: Generator over all possible glob expansions of a given path. Raises: ValueError: If given path is empty or relative. """ utils.AssertType(path, unicode) if not path: raise ValueError("Path is empty") drive, tail = os.path.splitdrive(path) if ((platform.system() == "Windows" and not drive) or not (tail and tail[0] == os.path.sep)): raise ValueError("Path '%s' is not absolute" % path) root_dir = os.path.join(drive, os.path.sep).upper() components = list(ParsePath(tail[1:], opts=opts)) return _ExpandComponents(root_dir, components)
def _CheckAccess(self, username, subject_id, approval_type): """Checks access to a given subject by a given user.""" utils.AssertType(subject_id, unicode) cache_key = (username, subject_id, approval_type) try: self.acl_cache.Get(cache_key) stats.STATS.IncrementCounter("approval_searches", fields=["-", "cache"]) return True except KeyError: stats.STATS.IncrementCounter("approval_searches", fields=["-", "reldb"]) approvals = data_store.REL_DB.ReadApprovalRequests( username, approval_type, subject_id=subject_id, include_expired=False) errors = [] for approval in approvals: try: approval_checks.CheckApprovalRequest(approval) self.acl_cache.Put(cache_key, True) return except access_control.UnauthorizedAccess as e: errors.append(e) subject = approval_checks.BuildLegacySubject(subject_id, approval_type) if not errors: raise access_control.UnauthorizedAccess( "No approval found.", subject=subject) else: raise access_control.UnauthorizedAccess( " ".join(utils.SmartStr(e) for e in errors), subject=subject)
def ParseFromUnicode(self, initializer): utils.AssertType(initializer, unicode) # Strip off the aff4: prefix if necessary. if initializer.startswith("aff4:/"): initializer = initializer[5:] self._string_urn = utils.NormalizePath(initializer)
def ExpandGroups(path): """Performs group expansion on a given path. For example, given path `foo/{bar,baz}/{quux,norf}` this method will yield `foo/bar/quux`, `foo/bar/norf`, `foo/baz/quux`, `foo/baz/norf`. Args: path: A path to expand. Yields: Paths that can be obtained from given path by expanding groups. """ utils.AssertType(path, unicode) chunks = [] offset = 0 for match in PATH_GROUP_REGEX.finditer(path): chunks.append([path[offset:match.start()]]) chunks.append(match.group("alts").split(",")) offset = match.end() chunks.append([path[offset:]]) for prod in itertools.product(*chunks): yield "".join(prod)
def _BuildStreamingResponse(self, binary_stream, method_name=None): """Builds HTTPResponse object for streaming.""" utils.AssertType(method_name, unicode) # We get a first chunk of the output stream. This way the likelihood # of catching an exception that may happen during response generation # is much higher. content = binary_stream.GenerateContent() try: peek = content.next() stream = itertools.chain([peek], content) except StopIteration: stream = [] response = werkzeug_wrappers.Response( response=stream, content_type="binary/octet-stream", direct_passthrough=True) response.headers["Content-Disposition"] = (( "attachment; filename=%s" % binary_stream.filename).encode("utf-8")) if method_name: response.headers["X-API-Method"] = method_name.encode("utf-8") if binary_stream.content_length: response.content_length = binary_stream.content_length return response
def ParseFromString(self, pem_string): utils.AssertType(pem_string, bytes) try: self._value = serialization.load_pem_public_key( pem_string, backend=openssl.backend) except (TypeError, ValueError, exceptions.UnsupportedAlgorithm) as e: raise type_info.TypeValueError("Public key invalid: %s" % e)
def ParseResponse(self, knowledge_base, response, path_type): del path_type # Unused. # TODO(hanuszczak): Why some of the registry value parsers anticipate string # response? This is stupid. utils.AssertType(response, (rdf_client_fs.StatEntry, rdfvalue.RDFString)) return self.Parse(response, knowledge_base)
def ParseFromDatastore(self, value): utils.AssertType(value, unicode) # TODO(hanuszczak): We should just assign the `self._string_urn` here # instead of including all of the parsing magic since the data store values # should be normalized already. But sadly this is not the case and for now # we have to deal with unnormalized values as well. self.ParseFromString(value)
def __init__(self, data=""): utils.AssertType(data, unicode) # Set the lexer up to process a new data feed. self.Reset() # Populate internal token list with class tokens, if defined. self._tokens = self.tokens[:] # Populate the lexer with any data we got. self.buffer = data
def ParseFromString(self, initializer): """Create RDFRUN from string. Args: initializer: url string """ utils.AssertType(initializer, bytes) self.ParseFromUnicode(initializer.decode("utf-8"))
def ParseEntries(self, data): utils.AssertType(data, unicode) self.entries = {} self.Reset() self.Feed(data) self.Close() found = set(self.entries) if self.required.issubset(found): return self.entries
def __init__(self, fopen=None): """Initializes the file parser object. Args: fopen: A function that returns file-like object for given pathspec. """ # TODO(hanuszczak): Define clear interface for file-like objects. if fopen is not None: utils.AssertType(fopen, types.FunctionType) self._fopen = fopen
def ParseFromHumanReadable(self, string): utils.AssertType(string, unicode) upper_string = string.upper() if upper_string == u"TRUE" or string == u"1": self._value = 1 elif upper_string == u"FALSE" or string == u"0": self._value = 0 else: raise ValueError("Unparsable boolean string: `%s`" % string)
def GenerateCSRFToken(user_id, time): """Generates a CSRF token based on a secret key, id and time.""" utils.AssertType(user_id, unicode) if time is not None: utils.AssertType(time, int) time = time or rdfvalue.RDFDatetime.Now().AsMicrosecondsSinceEpoch() secret = config.CONFIG.Get("AdminUI.csrf_secret_key", None) # TODO(amoser): Django is deprecated. Remove this at some point. if not secret: secret = config.CONFIG["AdminUI.django_secret_key"] digester = hmac.new(secret.encode("ascii"), digestmod=hashlib.sha256) digester.update(user_id.encode("ascii")) digester.update(CSRF_DELIMITER) digester.update(unicode(time).encode("ascii")) digest = digester.digest() token = base64.urlsafe_b64encode(b"%s%s%d" % (digest, CSRF_DELIMITER, time)) return token.rstrip(b"=")
def ParseResponse(self, knowledge_base, response, path_type): del path_type # Unused. utils.AssertType(response, rdf_client_action.ExecuteResponse) return self.Parse( cmd=response.request.cmd, args=response.request.args, stdout=response.stdout, stderr=response.stderr, return_val=response.exit_status, time_taken=response.time_used, knowledge_base=knowledge_base)
def Sign(self, message, use_pss=False): """Sign a given message.""" utils.AssertType(message, bytes) # TODO(amoser): This should use PSS by default at some point. if not use_pss: padding_algorithm = padding.PKCS1v15() else: padding_algorithm = padding.PSS( mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH) return self._value.sign(message, padding_algorithm, hashes.SHA256())
def ParseResponse(self, knowledge_base, response, path_type): del path_type # Unused. utils.AssertType(response, rdf_client_fs.StatEntry) # TODO(hanuszczak): This is a temporary hack to avoid rewriting hundreds of # tests for file parser. Once the tests are adapted to use the new API, the # constructor should be disallowed to take `None` as file opener as soon as # possible. if self._fopen is None: raise AssertionError("Parser constructed without file opening function") with self._fopen(response.pathspec) as filedesc: return self.Parse(response, filedesc, knowledge_base)
def ExpandPath(path, opts=None): """Applies all expansion mechanisms to the given path. Args: path: A path to expand. opts: A `PathOpts` object. Yields: All paths possible to obtain from a given path by performing expansions. """ utils.AssertType(path, unicode) for grouped_path in ExpandGroups(path): for globbed_path in ExpandGlobs(grouped_path, opts=opts): yield globbed_path
def __init__(self, client_id, client_mock, token=None): if not isinstance(client_id, rdf_client.ClientURN): client_id = rdf_client.ClientURN(client_id) if client_mock is None: client_mock = action_mocks.InvalidActionMock() else: utils.AssertType(client_mock, action_mocks.ActionMock) self._mock_task_queue = getattr(client_mock, "mock_task_queue", []) self.client_id = client_id self.client_mock = client_mock self.token = token # Well known flows are run on the front end. self.well_known_flows = flow.WellKnownFlow.GetAllWellKnownFlows(token=token)
def __init__(self, name, doc=None, args_type=None, result_type=None, category=None, http_methods=None, no_audit_log_required=False): utils.AssertType(name, unicode) self.name = name self.doc = doc self.args_type = args_type self.result_type = result_type self.category = category self.http_methods = http_methods or set() self.no_audit_log_required = no_audit_log_required
def _RunClientAction(self, action, request, parsers, path_type): """Runs the client action with the request and parses the result.""" responses = list(action(request)) if not parsers: return responses # filter parsers by process_together setting multi_parsers = [] single_parsers = [] for parser in parsers: # TODO(hanuszczak): This is absolutely disgusting and should be refactored # with some kind of factory, that produces different parser instances # depending whether we are on the server or on the client. Doing so would # probably solve issues with this very artificial metclass registry that # we have here. if issubclass(parser, parser_lib.FileParser): parser_obj = parser(vfs.VFSOpen) else: parser_obj = parser() if parser_obj.process_together: multi_parsers.append(parser_obj) else: single_parsers.append(parser_obj) # parse the responses parsed_responses = [] for response in responses: for parser in single_parsers: utils.AssertType(parser, parser_lib.SingleResponseParser) parsed_responses.extend( parser.ParseResponse(self.knowledge_base, response, path_type)) for parser in multi_parsers: for res in ParseMultipleResponses(parser, responses, self.knowledge_base, path_type): parsed_responses.append(res) return parsed_responses
def __init__(self, filename, content_generator=None, content_length=None): """ApiBinaryStream constructor. Args: filename: A file name to be used by the browser when user downloads the file. content_generator: A generator that yields byte chunks (of any size) to be streamed to the user. content_length: The length of the stream, if known upfront. Raises: ValueError: if content_generator is None. """ utils.AssertType(filename, unicode) self.filename = filename self.content_length = content_length if content_generator is None: raise ValueError("content_generator can't be None") self.content_generator = content_generator
def _AddToHostData(self, host_data, artifact, data, parser): """Parse raw data collected for an artifact into the host_data table.""" utils.AssertType(data, dict) rdfs = [] stats = [] files = [] for path, lines in iteritems(data): stat = self.CreateStat(path) stats.append(stat) file_obj = io.StringIO(lines) files.append(file_obj) if not parser.process_together: rdfs.extend(list(parser.Parse(stat, file_obj, None))) if parser.process_together: rdfs = list(parser.ParseMultiple(stats, files, None)) host_data[artifact] = self.SetArtifactData( anomaly=[a for a in rdfs if isinstance(a, rdf_anomaly.Anomaly)], parsed=[r for r in rdfs if not isinstance(r, rdf_anomaly.Anomaly)], raw=stats, results=host_data.get(artifact)) return host_data
def ParseFromString(self, pem_string): utils.AssertType(pem_string, bytes) try: self._value = serialization.load_pem_private_key( pem_string, password=None, backend=openssl.backend) return except (TypeError, ValueError, exceptions.UnsupportedAlgorithm) as e: if "private key is encrypted" not in str(e): raise type_info.TypeValueError("Private key invalid: %s" % e) # pylint: disable=g-explicit-bool-comparison, g-equals-none # The private key is passphrase protected, we need to see if we are # allowed to ask the user. # # If allow_prompt is False, we are explicitly told that we are not. if self.allow_prompt == False: raise type_info.TypeValueError("Private key invalid: %s" % e) # allow_prompt was not set, we use the context we are in to see if it # makes sense to ask. elif self.allow_prompt == None: # TODO(user): dependency loop with # core/grr_response_core/grr/config/client.py. # pylint: disable=protected-access if "Commandline Context" not in config_lib._CONFIG.context: raise type_info.TypeValueError("Private key invalid: %s" % e) # pylint: enable=protected-access # pylint: enable=g-explicit-bool-comparison, g-equals-none try: # The private key is encrypted and we can ask the user for the passphrase. password = utils.PassphraseCallback() self._value = serialization.load_pem_private_key( pem_string, password=password, backend=openssl.backend) except (TypeError, ValueError, exceptions.UnsupportedAlgorithm) as e: raise type_info.TypeValueError("Unable to load private key: %s" % e)
def ParsePath(path, opts=None): """Parses given path into a stream of `PathComponent` instances. Args: path: A path to be parsed. opts: An `PathOpts` object. Yields: `PathComponent` instances corresponding to the components of the given path. Raises: ValueError: If path contains more than one recursive component. """ utils.AssertType(path, unicode) rcount = 0 for item in path.split(os.path.sep): component = ParsePathItem(item, opts=opts) if isinstance(component, RecursiveComponent): rcount += 1 if rcount > 1: raise ValueError( "path cannot have more than one recursive component") yield component
def __init__(self, collection_id): utils.AssertType(collection_id, rdfvalue.RDFURN) super(SequentialCollection, self).__init__() # The collection_id for this collection is a RDFURN for now. self.collection_id = collection_id
def ParseFromHumanReadable(self, string): utils.AssertType(string, unicode) self._value = int(string)
def ParseFromDatastore(self, value): utils.AssertType(value, int) self._value = value