def Log(self, format_str, *args): """Logs the message using the flow's standard logging. Args: format_str: Format string *args: arguments to the format string Raises: ValueError: on parent missing logs_collection """ format_str = utils.SmartUnicode(format_str) status = format_str if args: try: # The status message is always in unicode status = format_str % args except TypeError: logging.error( "Tried to log a format string with the wrong number " "of arguments: %s", format_str) logging.info("%s: %s", self.session_id, status) self.SetStatus(utils.SmartUnicode(status)) log_entry = rdf_flows.FlowLog( client_id=self.runner_args.client_id, urn=self.session_id, flow_name=self.flow_obj.__class__.__name__, log_message=status) logs_collection_urn = self._GetLogCollectionURN( self.runner_args.logs_collection_urn) with data_store.DB.GetMutationPool() as pool: grr_collections.LogCollection.StaticAdd( logs_collection_urn, log_entry, mutation_pool=pool)
def Set(self, subject, attribute, value, timestamp=None, replace=True, sync=True): """Set the value into the data store.""" subject = utils.SmartUnicode(subject) _ = sync attribute = utils.SmartUnicode(attribute) if timestamp is None or timestamp == self.NEWEST_TIMESTAMP: timestamp = time.time() * 1000000 if subject not in self.subjects: self.subjects[subject] = {} if replace or attribute not in self.subjects[subject]: self.subjects[subject][attribute] = [] self.subjects[subject][attribute].append( [self._Encode(value), int(timestamp)]) self.subjects[subject][attribute].sort(key=lambda x: x[1])
def ProcessResponse(self, response): """Sends an email for each response.""" emails_left = self.IncrementCounter() if emails_left < 0: return client_id = response.source client = aff4.FACTORY.Open(client_id, token=self.token) hostname = client.Get(client.Schema.HOSTNAME) or "unknown hostname" client_fragment_id = "/clients/%s" % client_id.Basename() if emails_left == 0: additional_message = (self.too_many_mails_msg % self.state.args.emails_limit) else: additional_message = "" subject = self.__class__.subject_template.render( source_urn=utils.SmartUnicode(self.state.source_urn)) body = self.__class__.template.render( client_id=client_id, client_fragment_id=client_fragment_id, admin_ui_url=config.CONFIG["AdminUI.url"], source_urn=self.state.source_urn, additional_message=additional_message, signature=config.CONFIG["Email.signature"], hostname=utils.SmartUnicode(hostname), creator=utils.SmartUnicode(self.token.username)) email_alerts.EMAIL_ALERTER.SendEmail(self.state.args.email_address, "grr-noreply", utils.SmartStr(subject), utils.SmartStr(body), is_html=True)
def Run(self, unused_args): """Enumerates all the users on this system.""" users = self.ParseWtmp() for user, last_login in users.iteritems(): # Lose the null termination username = user.split("\x00", 1)[0] if username: # Somehow the last login time can be < 0. There is no documentation # what this means so we just set it to 0 (the rdfvalue field is # unsigned so we can't send negative values). if last_login < 0: last_login = 0 result = rdf_client.User(username=utils.SmartUnicode(username), last_logon=last_login * 1000000) try: pwdict = pwd.getpwnam(username) result.homedir = utils.SmartUnicode(pwdict.pw_dir) result.full_name = utils.SmartUnicode(pwdict.pw_gecos) result.uid = pwdict.pw_uid result.gid = pwdict.pw_gid result.shell = utils.SmartUnicode(pwdict.pw_shell) except KeyError: pass self.SendReply(result)
def Grant(self): """Create the Approval object and notify the Approval Granter.""" approvals_root_urn = aff4.ROOT_URN.Add("ACL").Add( self.subject_urn.Path()).Add(self.delegate) children_urns = list(aff4.FACTORY.ListChildren(approvals_root_urn)) if not children_urns: raise access_control.UnauthorizedAccess( "No approval found for user %s" % utils.SmartStr(self.token.username), subject=self.subject_urn) approvals = aff4.FACTORY.MultiOpen(children_urns, mode="r", aff4_type=Approval, token=self.token) found_approval_urn = None for approval in approvals: approval_reason = approval.Get(approval.Schema.REASON) if (utils.SmartUnicode(approval_reason) == utils.SmartUnicode( self.reason) and (not found_approval_urn or approval_reason.age > found_approval_urn.age)): found_approval_urn = approval.urn found_approval_urn.age = approval_reason.age if not found_approval_urn: raise access_control.UnauthorizedAccess( "No approval with reason '%s' found for user %s" % (utils.SmartStr( self.reason), utils.SmartStr(self.token.username)), subject=self.subject_urn) # This object must already exist. try: approval_request = aff4.FACTORY.Open(found_approval_urn, mode="rw", aff4_type=self.approval_type, token=self.token) except IOError: raise access_control.UnauthorizedAccess( "Approval object does not exist.", requested_access="rw") with approval_request: # We are now an approver for this request. approval_request.AddAttribute( approval_request.Schema.APPROVER(self.token.username)) return found_approval_urn
def CheckAccess(self, subject, token): """Checks for access to given subject with a given token. CheckAccess runs given subject through all "allow" clauses that were previously registered with Allow() calls. It returns True on first match and raises access_control.UnauthorizedAccess if there are no matches or if any of the additional checks fails. Args: subject: RDFURN of the subject that will be checked for access. token: User credentials token. Returns: True if access is granted. Raises: access_control.UnauthorizedAccess if access is rejected. """ subject = rdfvalue.RDFURN(subject) subject_str = subject.SerializeToString() for check_tuple in self.checks: regex_text, regex, require, require_args, require_kwargs = check_tuple match = regex.match(subject_str) if not match: continue if require: # If require() fails, it raises access_control.UnauthorizedAccess. require(subject, token, *require_args, **require_kwargs) logging.debug( u"Datastore access granted to %s on %s by pattern: %s " u"with reason: %s (require=%s, require_args=%s, " u"require_kwargs=%s, helper_name=%s)", utils.SmartUnicode(token.username), utils.SmartUnicode(subject_str), utils.SmartUnicode(regex_text), utils.SmartUnicode(token.reason), require, require_args, require_kwargs, self.helper_name) return True logging.warn("Datastore access denied to %s (no matched rules)", subject_str) raise access_control.UnauthorizedAccess( "Access to %s rejected: (no matched rules)." % subject, subject=subject)
def DeleteSubject(self, subject, sync=False): _ = sync subject = utils.SmartUnicode(subject) try: del self.subjects[subject] except KeyError: pass
def ListNames(self): directory_handle = self.fd.as_directory() for f in directory_handle: # TSK only deals with utf8 strings, but path components are always unicode # objects - so we convert to unicode as soon as we receive data from # TSK. Prefer to compare unicode objects to guarantee they are normalized. yield utils.SmartUnicode(f.info.name.name)
def testNoTraceOfDeletedHuntIsLeftInTheDataStore(self): # This only works with the test data store (FakeDataStore). if not isinstance(data_store.DB, fake_data_store.FakeDataStore): self.skipTest("Only supported on FakeDataStore.") with test_lib.ConfigOverrider( {"DataRetention.hunts_ttl": rdfvalue.Duration("1s")}): with test_lib.FakeTime(40 + 60 * self.NUM_HUNTS): flow.GRRFlow.StartFlow( flow_name=data_retention.CleanHunts.__name__, sync=True, token=self.token) for hunt_urn in self.hunts_urns: hunt_id = hunt_urn.Basename() for subject, subject_data in data_store.DB.subjects.items(): # Foreman rules are versioned, so hunt ids will be mentioned # there. Ignoring audit events as well. if subject == "aff4:/foreman" or subject.startswith( "aff4:/audit"): continue self.assertNotIn(hunt_id, subject) for column_name, values in subject_data.items(): self.assertNotIn(hunt_id, column_name) for value, _ in values: self.assertNotIn(hunt_id, utils.SmartUnicode(value))
def testNetgroupParser(self): """Ensure we can extract users from a netgroup file.""" parser = linux_file_parser.NetgroupParser() dat = u"""group1 (-,user1,) (-,user2,) (-,user3,) #group1 comment group2 (-,user4,) (-,user2,) super_group (-,user5,) (-,user6,) (-,文德文,) group1 group2 super_group2 (-,user7,) super_group super_group3 (-,user5,) (-,user6,) group1 group2 """ dat_fd = StringIO.StringIO(dat) with test_lib.ConfigOverrider( {"Artifacts.netgroup_user_blacklist": ["user2", "user3"]}): out = list(parser.Parse(None, dat_fd, None)) users = [] for result in out: if isinstance(result, rdf_anomaly.Anomaly): self.assertTrue( utils.SmartUnicode(u"文德文") in result.symptom) else: users.append(result) self.assertItemsEqual( [x.username for x in users], [u"user1", u"user4", u"user5", u"user6", u"user7"]) dat_fd.seek(0) with test_lib.ConfigOverrider( {"Artifacts.netgroup_filter_regexes": [r"^super_group3$"]}): out = list(parser.Parse(None, dat_fd, None)) self.assertItemsEqual([x.username for x in out], [u"user5", u"user6"])
def GetRawDevice(path): """Resolve the raw device that contains the path.""" device_map = GetMountpoints() path = utils.SmartUnicode(path) mount_point = path = utils.NormalizePath(path, "/") result = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS) # Assign the most specific mount point to the result while mount_point: try: result.path, fs_type = device_map[mount_point] if fs_type in [ "ext2", "ext3", "ext4", "vfat", "ntfs", "Apple_HFS", "hfs", "msdos" ]: # These are read filesystems result.pathtype = rdf_paths.PathSpec.PathType.OS else: result.pathtype = rdf_paths.PathSpec.PathType.UNSET # Drop the mount point path = utils.NormalizePath(path[len(mount_point):]) return result, path except KeyError: mount_point = os.path.dirname(mount_point)
def RemoveClientKeyword(self, client_id, keyword, cursor=None): """Removes the association of a particular client to a keyword.""" cursor.execute( "DELETE FROM client_keywords WHERE client_id=%s AND keyword=%s", [ mysql_utils.ClientIDToInt(client_id), utils.SmartUnicode(keyword) ])
def GetRawDevice(path): """Resolve the raw device that contains the path.""" device_map = GetMountpoints() path = utils.SmartUnicode(path) mount_point = path = utils.NormalizePath(path, "/") result = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS) # Assign the most specific mount point to the result while mount_point: try: result.path, fs_type = device_map[mount_point] if fs_type in SUPPORTED_FILESYSTEMS: # These are read filesystems result.pathtype = rdf_paths.PathSpec.PathType.OS else: logging.error( "Filesystem %s is not supported. Supported filesystems " "are %s", fs_type, SUPPORTED_FILESYSTEMS) result.pathtype = rdf_paths.PathSpec.PathType.UNSET # Drop the mount point path = utils.NormalizePath(path[len(mount_point):]) result.mount_point = mount_point return result, path except KeyError: mount_point = os.path.dirname(mount_point)
def testUnicodeListDirectory(self): """Test that the ListDirectory flow works on unicode directories.""" client_mock = action_mocks.ListDirectoryClientMock() # Deliberately specify incorrect casing pb = rdf_paths.PathSpec(path=os.path.join(self.base_path, "test_img.dd"), pathtype=rdf_paths.PathSpec.PathType.OS) pb.Append(path=u"入乡随俗 海外春节别样过法", pathtype=rdf_paths.PathSpec.PathType.TSK) flow_test_lib.TestFlowHelper(filesystem.ListDirectory.__name__, client_mock, client_id=self.client_id, pathspec=pb, token=self.token) # Check the output file is created output_path = self.client_id.Add("fs/tsk").Add(pb.CollapsePath()) fd = aff4.FACTORY.Open(output_path, token=self.token) children = list(fd.OpenChildren()) self.assertEqual(len(children), 1) child = children[0] self.assertEqual(os.path.basename(utils.SmartUnicode(child.urn)), u"入乡随俗.txt")
def Stat(self, path=None, ext_attrs=False): """Returns stat information of a specific path. Args: path: a Unicode string containing the path or None. If path is None the value in self.path is used. ext_attrs: Whether the call should also collect extended attributes. Returns: a StatResponse proto Raises: IOError when call to os.stat() fails """ # Note that the encoding of local path is system specific local_path = client_utils.CanonicalPathToLocalPath(path or self.path) result = client_utils.StatEntryFromPath( local_path, self.pathspec, ext_attrs=ext_attrs) # Is this a symlink? If so we need to note the real location of the file. try: result.symlink = utils.SmartUnicode(os.readlink(local_path)) except (OSError, AttributeError): pass return result
def Parse(self, stat, file_object, knowledge_base): """Parse the wtmp file.""" _, _ = stat, knowledge_base users = {} wtmp = file_object.read() while wtmp: try: record = UtmpStruct(wtmp) except utils.ParsingError: break wtmp = wtmp[record.size:] # Users only appear for USER_PROCESS events, others are system. if record.ut_type != 7: continue # Lose the null termination record.user = record.user.split("\x00", 1)[0] # Store the latest login time. # TODO(user): remove the 0 here once RDFDatetime can support times # pre-epoch properly. try: users[record.user] = max(users[record.user], record.sec, 0) except KeyError: users[record.user] = record.sec for user, last_login in users.iteritems(): yield rdf_client.User(username=utils.SmartUnicode(user), last_logon=last_login * 1000000)
def ScanAttributes(self, subject_prefix, attributes, after_urn="", max_records=None, relaxed_order=False): subject_prefix = utils.SmartStr(rdfvalue.RDFURN(subject_prefix)) if subject_prefix[-1] != "/": subject_prefix += "/" if after_urn: after_urn = utils.SmartUnicode(after_urn) subjects = [] for s in self.subjects.keys(): if s.startswith(subject_prefix) and s > after_urn: subjects.append(s) subjects.sort() return_count = 0 for s in subjects: if max_records and return_count >= max_records: break r = self.subjects[s] results = {} for attribute in attributes: attribute_list = r.get(attribute) if attribute_list: value, timestamp = attribute_list[-1] results[attribute] = (timestamp, value) if results: return_count += 1 yield (s, results)
def __init__(self, *children, **kwargs): super(Regexp, self).__init__(*children, **kwargs) try: self.compiled_re = re.compile(utils.SmartUnicode(self.right_operand)) except re.error: raise ValueError( "Regular expression \"%s\" is malformed." % self.right_operand)
def Parse(self, stat, knowledge_base): """Parse each returned registry value.""" _ = knowledge_base # Unused. sid_str = stat.pathspec.Dirname().Basename() if SID_RE.match(sid_str): kb_user = rdf_client.User() kb_user.sid = sid_str if stat.pathspec.Basename() == "ProfileImagePath": if stat.resident: # Support old clients. kb_user.homedir = utils.SmartUnicode(stat.resident) else: kb_user.homedir = stat.registry_data.GetValue() kb_user.userprofile = kb_user.homedir try: # Assume username is the last component of the path. This is not # robust, but other user artifacts will override it if there is a # better match. kb_user.username = kb_user.homedir.rsplit("\\", 1)[-1] except IndexError: pass yield kb_user
def MultiResolvePrefix(self, subjects, attribute_prefix, timestamp=None, limit=None): unicode_to_orig = {utils.SmartUnicode(s): s for s in subjects} result = {} for unicode_subject, orig_subject in unicode_to_orig.iteritems(): values = self.ResolvePrefix(unicode_subject, attribute_prefix, timestamp=timestamp, limit=limit) if not values: continue if limit: if limit < len(values): values = values[:limit] result[orig_subject] = values limit -= len(values) if limit <= 0: return result.iteritems() else: result[orig_subject] = values return result.iteritems()
def BuildToken(request, execution_time): """Build an ACLToken from the request.""" # The request.args dictionary will also be filled on HEAD calls. if request.method in ["GET", "HEAD"]: reason = request.args.get("reason", "") elif request.method in ["POST", "DELETE", "PATCH"]: # The header X-GRR-Reason is set in api-service.js. reason = utils.SmartUnicode( urllib2.unquote(request.headers.get("X-Grr-Reason", ""))) # We assume that request.user contains the username that we can trust. # No matter what authentication method is used, the WebAuthManager is # responsible for authenticating the userand setting request.user to # a correct value (see gui/webauth.py). # # The token that's built here will be later used to find an API router, # get the ApiCallHandler from the router, and then to call the handler's # Handle() method. API router will be responsible for all the ACL checks. token = access_control.ACLToken(username=request.user, reason=reason, process="GRRAdminUI", expiry=rdfvalue.RDFDatetime.Now() + execution_time) for field in ["Remote_Addr", "X-Forwarded-For"]: remote_addr = request.headers.get(field, "") if remote_addr: token.source_ips.append(remote_addr) return token
def ProcessMessage(self, message=None): """Processes this event.""" client_id = message.source message = message.payload.string logging.info(self.logline, client_id, message) # Write crash data. if data_store.RelationalDBReadEnabled(): client = data_store.REL_DB.ReadClientSnapshot(client_id) client_info = client.startup_info.client_info else: client = aff4.FACTORY.Open(client_id, token=self.token) client_info = client.Get(client.Schema.CLIENT_INFO) crash_details = rdf_client.ClientCrash(client_id=client_id, client_info=client_info, crash_message=message, timestamp=long(time.time() * 1e6), crash_type="Nanny Message") self.WriteAllCrashDetails(client_id, crash_details, token=self.token) # Also send email. if config.CONFIG["Monitoring.alert_email"]: client = aff4.FACTORY.Open(client_id, token=self.token) hostname = client.Get(client.Schema.HOSTNAME) url = "/clients/%s" % client_id.Basename() body = self.__class__.mail_template.render( client_id=client_id, admin_ui=config.CONFIG["AdminUI.url"], hostname=utils.SmartUnicode(hostname), signature=config.CONFIG["Email.signature"], url=url, message=utils.SmartUnicode(message)) email_alerts.EMAIL_ALERTER.SendEmail( config.CONFIG["Monitoring.alert_email"], "GRR server", self.subject % client_id, utils.SmartStr(body), is_html=True)
def AddClientLabels(self, client_id, owner, labels): """Attaches a user label to a client.""" if client_id not in self.metadatas: raise db.UnknownClientError(client_id) labelset = self.labels.setdefault(client_id, {}).setdefault(owner, set()) for l in labels: labelset.add(utils.SmartUnicode(l))
def SetCoreGRRKnowledgeBaseValues(kb, client_obj): """Set core values from GRR into the knowledgebase.""" client_schema = client_obj.Schema kb.fqdn = utils.SmartUnicode(client_obj.Get(client_schema.FQDN, "")) if not kb.fqdn: kb.fqdn = utils.SmartUnicode(client_obj.Get(client_schema.HOSTNAME, "")) versions = client_obj.Get(client_schema.OS_VERSION) if versions and versions.versions: try: kb.os_major_version = versions.versions[0] kb.os_minor_version = versions.versions[1] except IndexError: # Some OSs don't have a minor version. pass client_os = client_obj.Get(client_schema.SYSTEM) if client_os: kb.os = utils.SmartUnicode(client_obj.Get(client_schema.SYSTEM))
def _ImportRow(store, row, product_code_list, op_system_code_list): sha1 = row[0].lower() md5 = row[1].lower() crc = int(row[2].lower(), 16) file_name = utils.SmartUnicode(row[3]) file_size = int(row[4]) special_code = row[7] store.AddHash(sha1, md5, crc, file_name, file_size, product_code_list, op_system_code_list, special_code)
def ParseMultiple(self, stats, knowledge_base): """Parse Service registry keys and return WindowsServiceInformation.""" _ = knowledge_base services = {} field_map = { "Description": "description", "DisplayName": "display_name", "Group": "group_name", "DriverPackageId": "driver_package_id", "ErrorControl": "error_control", "ImagePath": "image_path", "ObjectName": "object_name", "Start": "startup_type", "Type": "service_type", "Parameters/ServiceDLL": "service_dll" } # Field map key should be converted to lowercase because key aquired through # self._GetKeyName could have some characters in different case than the # field map, e.g. ServiceDLL and ServiceDll. field_map = {k.lower(): v for k, v in field_map.items()} for stat in stats: # Ignore subkeys if not stat.HasField("registry_data"): continue service_name = self._GetServiceName(stat.pathspec.path) reg_key = os.path.dirname(stat.pathspec.path) service_info = rdf_client.WindowsServiceInformation( name=service_name, registry_key=reg_key) services.setdefault(service_name, service_info) key = self._GetKeyName(stat.pathspec.path) if key in field_map: try: services[service_name].Set(field_map[key], stat.registry_data.GetValue()) except type_info.TypeValueError: # Flatten multi strings into a simple string if (stat.registry_type == rdf_client.StatEntry.RegistryType.REG_MULTI_SZ): services[service_name].Set( field_map[key], utils.SmartUnicode(stat.registry_data.GetValue())) else: # Log failures for everything else # TODO(user): change this to yield a ParserAnomaly object. dest_type = type(services[service_name].Get(field_map[key])) logging.debug("Wrong type set for %s:%s, expected %s, got %s", stat.pathspec.path, stat.registry_data.GetValue(), dest_type, type(stat.registry_data.GetValue())) return services.itervalues()
def InsertArg(self, string="", **_): """Insert an arg to the current expression.""" if self.verbose: logging.debug("Storing Argument %s", utils.SmartUnicode(string)) # This expression is complete if self.current_expression.AddArg(string): self.stack.append(self.current_expression) self.current_expression = self.expression_cls() return self.PopState()
def Publish(self, event_name, message): """Publish a message to an event queue. Args: event_name: The name of the event to publish to. message: An RDFValue instance to publish to the event listeners. """ logging.debug("Publishing %s to %s", utils.SmartUnicode(message)[:100], event_name) events.Events.PublishEvent(event_name, message, token=self.token)
def _Decode(self, attribute, value): required_type = self._attribute_types.get(attribute, "bytes") if isinstance(value, buffer): value = str(value) if required_type in ("integer", "unsigned_integer"): return int(value) elif required_type == "string": return utils.SmartUnicode(value) else: return value
def ResolveMulti(self, subject, attributes, timestamp=None, limit=None): subject = utils.SmartUnicode(subject) # Does timestamp represent a range? if isinstance(timestamp, (list, tuple)): start, end = timestamp # pylint: disable=unpacking-non-sequence else: start, end = -1, 1 << 65 start = int(start) end = int(end) if isinstance(attributes, str): attributes = [attributes] try: record = self.subjects[subject] except KeyError: return # Holds all the attributes which matched. Keys are attribute names, values # are lists of timestamped data. results = {} for attribute in attributes: for attr, values in record.iteritems(): if attr == attribute: for value, ts in values: results_list = results.setdefault(attribute, []) # If we are always after the latest ts we clear older ones. if (results_list and timestamp == self.NEWEST_TIMESTAMP and results_list[0][1] < ts): results_list = [] results[attribute] = results_list # Timestamp outside the range, drop it. elif ts < start or ts > end: continue results_list.append((attribute, ts, value)) # Return the results in the same order they requested. remaining_limit = limit for attribute in attributes: # This returns triples of (attribute_name, timestamp, data). We want to # sort by timestamp. for _, ts, data in sorted(results.get(attribute, []), key=lambda x: x[1], reverse=True): if remaining_limit: remaining_limit -= 1 if remaining_limit == 0: yield (attribute, data, ts) return yield (attribute, data, ts)