Beispiel #1
0
  def Log(self, format_str, *args):
    """Logs the message using the flow's standard logging.

    Args:
      format_str: Format string
      *args: arguments to the format string
    Raises:
      ValueError: on parent missing logs_collection
    """
    format_str = utils.SmartUnicode(format_str)

    status = format_str
    if args:
      try:
        # The status message is always in unicode
        status = format_str % args
      except TypeError:
        logging.error(
            "Tried to log a format string with the wrong number "
            "of arguments: %s", format_str)

    logging.info("%s: %s", self.session_id, status)

    self.SetStatus(utils.SmartUnicode(status))

    log_entry = rdf_flows.FlowLog(
        client_id=self.runner_args.client_id,
        urn=self.session_id,
        flow_name=self.flow_obj.__class__.__name__,
        log_message=status)
    logs_collection_urn = self._GetLogCollectionURN(
        self.runner_args.logs_collection_urn)
    with data_store.DB.GetMutationPool() as pool:
      grr_collections.LogCollection.StaticAdd(
          logs_collection_urn, log_entry, mutation_pool=pool)
Beispiel #2
0
    def Set(self,
            subject,
            attribute,
            value,
            timestamp=None,
            replace=True,
            sync=True):
        """Set the value into the data store."""
        subject = utils.SmartUnicode(subject)

        _ = sync
        attribute = utils.SmartUnicode(attribute)

        if timestamp is None or timestamp == self.NEWEST_TIMESTAMP:
            timestamp = time.time() * 1000000

        if subject not in self.subjects:
            self.subjects[subject] = {}

        if replace or attribute not in self.subjects[subject]:
            self.subjects[subject][attribute] = []

        self.subjects[subject][attribute].append(
            [self._Encode(value), int(timestamp)])
        self.subjects[subject][attribute].sort(key=lambda x: x[1])
Beispiel #3
0
    def ProcessResponse(self, response):
        """Sends an email for each response."""
        emails_left = self.IncrementCounter()
        if emails_left < 0:
            return

        client_id = response.source
        client = aff4.FACTORY.Open(client_id, token=self.token)
        hostname = client.Get(client.Schema.HOSTNAME) or "unknown hostname"
        client_fragment_id = "/clients/%s" % client_id.Basename()

        if emails_left == 0:
            additional_message = (self.too_many_mails_msg %
                                  self.state.args.emails_limit)
        else:
            additional_message = ""

        subject = self.__class__.subject_template.render(
            source_urn=utils.SmartUnicode(self.state.source_urn))
        body = self.__class__.template.render(
            client_id=client_id,
            client_fragment_id=client_fragment_id,
            admin_ui_url=config.CONFIG["AdminUI.url"],
            source_urn=self.state.source_urn,
            additional_message=additional_message,
            signature=config.CONFIG["Email.signature"],
            hostname=utils.SmartUnicode(hostname),
            creator=utils.SmartUnicode(self.token.username))

        email_alerts.EMAIL_ALERTER.SendEmail(self.state.args.email_address,
                                             "grr-noreply",
                                             utils.SmartStr(subject),
                                             utils.SmartStr(body),
                                             is_html=True)
Beispiel #4
0
    def Run(self, unused_args):
        """Enumerates all the users on this system."""
        users = self.ParseWtmp()
        for user, last_login in users.iteritems():

            # Lose the null termination
            username = user.split("\x00", 1)[0]

            if username:
                # Somehow the last login time can be < 0. There is no documentation
                # what this means so we just set it to 0 (the rdfvalue field is
                # unsigned so we can't send negative values).
                if last_login < 0:
                    last_login = 0

                result = rdf_client.User(username=utils.SmartUnicode(username),
                                         last_logon=last_login * 1000000)

                try:
                    pwdict = pwd.getpwnam(username)
                    result.homedir = utils.SmartUnicode(pwdict.pw_dir)
                    result.full_name = utils.SmartUnicode(pwdict.pw_gecos)
                    result.uid = pwdict.pw_uid
                    result.gid = pwdict.pw_gid
                    result.shell = utils.SmartUnicode(pwdict.pw_shell)
                except KeyError:
                    pass

                self.SendReply(result)
Beispiel #5
0
    def Grant(self):
        """Create the Approval object and notify the Approval Granter."""

        approvals_root_urn = aff4.ROOT_URN.Add("ACL").Add(
            self.subject_urn.Path()).Add(self.delegate)
        children_urns = list(aff4.FACTORY.ListChildren(approvals_root_urn))
        if not children_urns:
            raise access_control.UnauthorizedAccess(
                "No approval found for user %s" %
                utils.SmartStr(self.token.username),
                subject=self.subject_urn)

        approvals = aff4.FACTORY.MultiOpen(children_urns,
                                           mode="r",
                                           aff4_type=Approval,
                                           token=self.token)
        found_approval_urn = None
        for approval in approvals:
            approval_reason = approval.Get(approval.Schema.REASON)
            if (utils.SmartUnicode(approval_reason) == utils.SmartUnicode(
                    self.reason)
                    and (not found_approval_urn
                         or approval_reason.age > found_approval_urn.age)):
                found_approval_urn = approval.urn
                found_approval_urn.age = approval_reason.age

        if not found_approval_urn:
            raise access_control.UnauthorizedAccess(
                "No approval with reason '%s' found for user %s" %
                (utils.SmartStr(
                    self.reason), utils.SmartStr(self.token.username)),
                subject=self.subject_urn)

        # This object must already exist.
        try:
            approval_request = aff4.FACTORY.Open(found_approval_urn,
                                                 mode="rw",
                                                 aff4_type=self.approval_type,
                                                 token=self.token)
        except IOError:
            raise access_control.UnauthorizedAccess(
                "Approval object does not exist.", requested_access="rw")

        with approval_request:
            # We are now an approver for this request.
            approval_request.AddAttribute(
                approval_request.Schema.APPROVER(self.token.username))

        return found_approval_urn
Beispiel #6
0
    def CheckAccess(self, subject, token):
        """Checks for access to given subject with a given token.

    CheckAccess runs given subject through all "allow" clauses that
    were previously registered with Allow() calls. It returns True on
    first match and raises access_control.UnauthorizedAccess if there
    are no matches or if any of the additional checks fails.

    Args:
      subject: RDFURN of the subject that will be checked for access.
      token: User credentials token.

    Returns:
      True if access is granted.

    Raises:
      access_control.UnauthorizedAccess if access is rejected.
    """
        subject = rdfvalue.RDFURN(subject)
        subject_str = subject.SerializeToString()

        for check_tuple in self.checks:
            regex_text, regex, require, require_args, require_kwargs = check_tuple

            match = regex.match(subject_str)
            if not match:
                continue

            if require:
                # If require() fails, it raises access_control.UnauthorizedAccess.
                require(subject, token, *require_args, **require_kwargs)

            logging.debug(
                u"Datastore access granted to %s on %s by pattern: %s "
                u"with reason: %s (require=%s, require_args=%s, "
                u"require_kwargs=%s, helper_name=%s)",
                utils.SmartUnicode(token.username),
                utils.SmartUnicode(subject_str),
                utils.SmartUnicode(regex_text),
                utils.SmartUnicode(token.reason), require, require_args,
                require_kwargs, self.helper_name)
            return True

        logging.warn("Datastore access denied to %s (no matched rules)",
                     subject_str)
        raise access_control.UnauthorizedAccess(
            "Access to %s rejected: (no matched rules)." % subject,
            subject=subject)
Beispiel #7
0
 def DeleteSubject(self, subject, sync=False):
     _ = sync
     subject = utils.SmartUnicode(subject)
     try:
         del self.subjects[subject]
     except KeyError:
         pass
Beispiel #8
0
 def ListNames(self):
     directory_handle = self.fd.as_directory()
     for f in directory_handle:
         # TSK only deals with utf8 strings, but path components are always unicode
         # objects - so we convert to unicode as soon as we receive data from
         # TSK. Prefer to compare unicode objects to guarantee they are normalized.
         yield utils.SmartUnicode(f.info.name.name)
Beispiel #9
0
    def testNoTraceOfDeletedHuntIsLeftInTheDataStore(self):
        # This only works with the test data store (FakeDataStore).
        if not isinstance(data_store.DB, fake_data_store.FakeDataStore):
            self.skipTest("Only supported on FakeDataStore.")

        with test_lib.ConfigOverrider(
            {"DataRetention.hunts_ttl": rdfvalue.Duration("1s")}):
            with test_lib.FakeTime(40 + 60 * self.NUM_HUNTS):
                flow.GRRFlow.StartFlow(
                    flow_name=data_retention.CleanHunts.__name__,
                    sync=True,
                    token=self.token)

            for hunt_urn in self.hunts_urns:
                hunt_id = hunt_urn.Basename()

                for subject, subject_data in data_store.DB.subjects.items():
                    # Foreman rules are versioned, so hunt ids will be mentioned
                    # there. Ignoring audit events as well.
                    if subject == "aff4:/foreman" or subject.startswith(
                            "aff4:/audit"):
                        continue

                    self.assertNotIn(hunt_id, subject)

                    for column_name, values in subject_data.items():
                        self.assertNotIn(hunt_id, column_name)

                        for value, _ in values:
                            self.assertNotIn(hunt_id,
                                             utils.SmartUnicode(value))
Beispiel #10
0
    def testNetgroupParser(self):
        """Ensure we can extract users from a netgroup file."""
        parser = linux_file_parser.NetgroupParser()
        dat = u"""group1 (-,user1,) (-,user2,) (-,user3,)
#group1 comment
group2 (-,user4,) (-,user2,)

super_group (-,user5,) (-,user6,) (-,文德文,) group1 group2
super_group2 (-,user7,) super_group
super_group3 (-,user5,) (-,user6,) group1 group2
"""
        dat_fd = StringIO.StringIO(dat)

        with test_lib.ConfigOverrider(
            {"Artifacts.netgroup_user_blacklist": ["user2", "user3"]}):
            out = list(parser.Parse(None, dat_fd, None))
            users = []
            for result in out:
                if isinstance(result, rdf_anomaly.Anomaly):
                    self.assertTrue(
                        utils.SmartUnicode(u"文德文") in result.symptom)
                else:
                    users.append(result)

            self.assertItemsEqual(
                [x.username for x in users],
                [u"user1", u"user4", u"user5", u"user6", u"user7"])

            dat_fd.seek(0)

        with test_lib.ConfigOverrider(
            {"Artifacts.netgroup_filter_regexes": [r"^super_group3$"]}):
            out = list(parser.Parse(None, dat_fd, None))
            self.assertItemsEqual([x.username for x in out],
                                  [u"user5", u"user6"])
Beispiel #11
0
def GetRawDevice(path):
  """Resolve the raw device that contains the path."""
  device_map = GetMountpoints()

  path = utils.SmartUnicode(path)
  mount_point = path = utils.NormalizePath(path, "/")

  result = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS)

  # Assign the most specific mount point to the result
  while mount_point:
    try:
      result.path, fs_type = device_map[mount_point]
      if fs_type in [
          "ext2", "ext3", "ext4", "vfat", "ntfs", "Apple_HFS", "hfs", "msdos"
      ]:
        # These are read filesystems
        result.pathtype = rdf_paths.PathSpec.PathType.OS
      else:
        result.pathtype = rdf_paths.PathSpec.PathType.UNSET

      # Drop the mount point
      path = utils.NormalizePath(path[len(mount_point):])

      return result, path
    except KeyError:
      mount_point = os.path.dirname(mount_point)
Beispiel #12
0
 def RemoveClientKeyword(self, client_id, keyword, cursor=None):
     """Removes the association of a particular client to a keyword."""
     cursor.execute(
         "DELETE FROM client_keywords WHERE client_id=%s AND keyword=%s", [
             mysql_utils.ClientIDToInt(client_id),
             utils.SmartUnicode(keyword)
         ])
Beispiel #13
0
def GetRawDevice(path):
    """Resolve the raw device that contains the path."""
    device_map = GetMountpoints()

    path = utils.SmartUnicode(path)
    mount_point = path = utils.NormalizePath(path, "/")

    result = rdf_paths.PathSpec(pathtype=rdf_paths.PathSpec.PathType.OS)

    # Assign the most specific mount point to the result
    while mount_point:
        try:
            result.path, fs_type = device_map[mount_point]
            if fs_type in SUPPORTED_FILESYSTEMS:
                # These are read filesystems
                result.pathtype = rdf_paths.PathSpec.PathType.OS
            else:
                logging.error(
                    "Filesystem %s is not supported. Supported filesystems "
                    "are %s", fs_type, SUPPORTED_FILESYSTEMS)
                result.pathtype = rdf_paths.PathSpec.PathType.UNSET

            # Drop the mount point
            path = utils.NormalizePath(path[len(mount_point):])
            result.mount_point = mount_point

            return result, path
        except KeyError:
            mount_point = os.path.dirname(mount_point)
Beispiel #14
0
    def testUnicodeListDirectory(self):
        """Test that the ListDirectory flow works on unicode directories."""

        client_mock = action_mocks.ListDirectoryClientMock()

        # Deliberately specify incorrect casing
        pb = rdf_paths.PathSpec(path=os.path.join(self.base_path,
                                                  "test_img.dd"),
                                pathtype=rdf_paths.PathSpec.PathType.OS)

        pb.Append(path=u"入乡随俗 海外春节别样过法",
                  pathtype=rdf_paths.PathSpec.PathType.TSK)

        flow_test_lib.TestFlowHelper(filesystem.ListDirectory.__name__,
                                     client_mock,
                                     client_id=self.client_id,
                                     pathspec=pb,
                                     token=self.token)

        # Check the output file is created
        output_path = self.client_id.Add("fs/tsk").Add(pb.CollapsePath())

        fd = aff4.FACTORY.Open(output_path, token=self.token)
        children = list(fd.OpenChildren())
        self.assertEqual(len(children), 1)
        child = children[0]
        self.assertEqual(os.path.basename(utils.SmartUnicode(child.urn)),
                         u"入乡随俗.txt")
Beispiel #15
0
  def Stat(self, path=None, ext_attrs=False):
    """Returns stat information of a specific path.

    Args:
      path: a Unicode string containing the path or None.
            If path is None the value in self.path is used.
      ext_attrs: Whether the call should also collect extended attributes.

    Returns:
      a StatResponse proto

    Raises:
      IOError when call to os.stat() fails
    """
    # Note that the encoding of local path is system specific
    local_path = client_utils.CanonicalPathToLocalPath(path or self.path)
    result = client_utils.StatEntryFromPath(
        local_path, self.pathspec, ext_attrs=ext_attrs)

    # Is this a symlink? If so we need to note the real location of the file.
    try:
      result.symlink = utils.SmartUnicode(os.readlink(local_path))
    except (OSError, AttributeError):
      pass

    return result
Beispiel #16
0
    def Parse(self, stat, file_object, knowledge_base):
        """Parse the wtmp file."""
        _, _ = stat, knowledge_base
        users = {}
        wtmp = file_object.read()
        while wtmp:
            try:
                record = UtmpStruct(wtmp)
            except utils.ParsingError:
                break

            wtmp = wtmp[record.size:]
            # Users only appear for USER_PROCESS events, others are system.
            if record.ut_type != 7:
                continue

            # Lose the null termination
            record.user = record.user.split("\x00", 1)[0]

            # Store the latest login time.
            # TODO(user): remove the 0 here once RDFDatetime can support times
            # pre-epoch properly.
            try:
                users[record.user] = max(users[record.user], record.sec, 0)
            except KeyError:
                users[record.user] = record.sec

        for user, last_login in users.iteritems():
            yield rdf_client.User(username=utils.SmartUnicode(user),
                                  last_logon=last_login * 1000000)
Beispiel #17
0
    def ScanAttributes(self,
                       subject_prefix,
                       attributes,
                       after_urn="",
                       max_records=None,
                       relaxed_order=False):
        subject_prefix = utils.SmartStr(rdfvalue.RDFURN(subject_prefix))
        if subject_prefix[-1] != "/":
            subject_prefix += "/"
        if after_urn:
            after_urn = utils.SmartUnicode(after_urn)
        subjects = []
        for s in self.subjects.keys():
            if s.startswith(subject_prefix) and s > after_urn:
                subjects.append(s)
        subjects.sort()

        return_count = 0
        for s in subjects:
            if max_records and return_count >= max_records:
                break
            r = self.subjects[s]
            results = {}
            for attribute in attributes:
                attribute_list = r.get(attribute)
                if attribute_list:
                    value, timestamp = attribute_list[-1]
                    results[attribute] = (timestamp, value)
            if results:
                return_count += 1
                yield (s, results)
Beispiel #18
0
 def __init__(self, *children, **kwargs):
   super(Regexp, self).__init__(*children, **kwargs)
   try:
     self.compiled_re = re.compile(utils.SmartUnicode(self.right_operand))
   except re.error:
     raise ValueError(
         "Regular expression \"%s\" is malformed." % self.right_operand)
Beispiel #19
0
  def Parse(self, stat, knowledge_base):
    """Parse each returned registry value."""
    _ = knowledge_base  # Unused.
    sid_str = stat.pathspec.Dirname().Basename()

    if SID_RE.match(sid_str):
      kb_user = rdf_client.User()
      kb_user.sid = sid_str
      if stat.pathspec.Basename() == "ProfileImagePath":
        if stat.resident:
          # Support old clients.
          kb_user.homedir = utils.SmartUnicode(stat.resident)
        else:
          kb_user.homedir = stat.registry_data.GetValue()

        kb_user.userprofile = kb_user.homedir
        try:
          # Assume username is the last component of the path. This is not
          # robust, but other user artifacts will override it if there is a
          # better match.
          kb_user.username = kb_user.homedir.rsplit("\\", 1)[-1]
        except IndexError:
          pass

      yield kb_user
Beispiel #20
0
    def MultiResolvePrefix(self,
                           subjects,
                           attribute_prefix,
                           timestamp=None,
                           limit=None):
        unicode_to_orig = {utils.SmartUnicode(s): s for s in subjects}
        result = {}
        for unicode_subject, orig_subject in unicode_to_orig.iteritems():

            values = self.ResolvePrefix(unicode_subject,
                                        attribute_prefix,
                                        timestamp=timestamp,
                                        limit=limit)

            if not values:
                continue

            if limit:
                if limit < len(values):
                    values = values[:limit]
                result[orig_subject] = values
                limit -= len(values)
                if limit <= 0:
                    return result.iteritems()
            else:
                result[orig_subject] = values

        return result.iteritems()
Beispiel #21
0
    def BuildToken(request, execution_time):
        """Build an ACLToken from the request."""

        # The request.args dictionary will also be filled on HEAD calls.
        if request.method in ["GET", "HEAD"]:
            reason = request.args.get("reason", "")
        elif request.method in ["POST", "DELETE", "PATCH"]:
            # The header X-GRR-Reason is set in api-service.js.
            reason = utils.SmartUnicode(
                urllib2.unquote(request.headers.get("X-Grr-Reason", "")))

        # We assume that request.user contains the username that we can trust.
        # No matter what authentication method is used, the WebAuthManager is
        # responsible for authenticating the userand setting request.user to
        # a correct value (see gui/webauth.py).
        #
        # The token that's built here will be later used to find an API router,
        # get the ApiCallHandler from the router, and then to call the handler's
        # Handle() method. API router will be responsible for all the ACL checks.
        token = access_control.ACLToken(username=request.user,
                                        reason=reason,
                                        process="GRRAdminUI",
                                        expiry=rdfvalue.RDFDatetime.Now() +
                                        execution_time)

        for field in ["Remote_Addr", "X-Forwarded-For"]:
            remote_addr = request.headers.get(field, "")
            if remote_addr:
                token.source_ips.append(remote_addr)
        return token
Beispiel #22
0
    def ProcessMessage(self, message=None):
        """Processes this event."""

        client_id = message.source

        message = message.payload.string

        logging.info(self.logline, client_id, message)

        # Write crash data.
        if data_store.RelationalDBReadEnabled():
            client = data_store.REL_DB.ReadClientSnapshot(client_id)
            client_info = client.startup_info.client_info
        else:
            client = aff4.FACTORY.Open(client_id, token=self.token)
            client_info = client.Get(client.Schema.CLIENT_INFO)

        crash_details = rdf_client.ClientCrash(client_id=client_id,
                                               client_info=client_info,
                                               crash_message=message,
                                               timestamp=long(time.time() *
                                                              1e6),
                                               crash_type="Nanny Message")

        self.WriteAllCrashDetails(client_id, crash_details, token=self.token)

        # Also send email.
        if config.CONFIG["Monitoring.alert_email"]:
            client = aff4.FACTORY.Open(client_id, token=self.token)
            hostname = client.Get(client.Schema.HOSTNAME)
            url = "/clients/%s" % client_id.Basename()

            body = self.__class__.mail_template.render(
                client_id=client_id,
                admin_ui=config.CONFIG["AdminUI.url"],
                hostname=utils.SmartUnicode(hostname),
                signature=config.CONFIG["Email.signature"],
                url=url,
                message=utils.SmartUnicode(message))
            email_alerts.EMAIL_ALERTER.SendEmail(
                config.CONFIG["Monitoring.alert_email"],
                "GRR server",
                self.subject % client_id,
                utils.SmartStr(body),
                is_html=True)
Beispiel #23
0
    def AddClientLabels(self, client_id, owner, labels):
        """Attaches a user label to a client."""
        if client_id not in self.metadatas:
            raise db.UnknownClientError(client_id)

        labelset = self.labels.setdefault(client_id,
                                          {}).setdefault(owner, set())
        for l in labels:
            labelset.add(utils.SmartUnicode(l))
Beispiel #24
0
def SetCoreGRRKnowledgeBaseValues(kb, client_obj):
    """Set core values from GRR into the knowledgebase."""
    client_schema = client_obj.Schema
    kb.fqdn = utils.SmartUnicode(client_obj.Get(client_schema.FQDN, ""))
    if not kb.fqdn:
        kb.fqdn = utils.SmartUnicode(client_obj.Get(client_schema.HOSTNAME,
                                                    ""))
    versions = client_obj.Get(client_schema.OS_VERSION)
    if versions and versions.versions:
        try:
            kb.os_major_version = versions.versions[0]
            kb.os_minor_version = versions.versions[1]
        except IndexError:
            # Some OSs don't have a minor version.
            pass
    client_os = client_obj.Get(client_schema.SYSTEM)
    if client_os:
        kb.os = utils.SmartUnicode(client_obj.Get(client_schema.SYSTEM))
Beispiel #25
0
def _ImportRow(store, row, product_code_list, op_system_code_list):
    sha1 = row[0].lower()
    md5 = row[1].lower()
    crc = int(row[2].lower(), 16)
    file_name = utils.SmartUnicode(row[3])
    file_size = int(row[4])
    special_code = row[7]
    store.AddHash(sha1, md5, crc, file_name, file_size, product_code_list,
                  op_system_code_list, special_code)
Beispiel #26
0
  def ParseMultiple(self, stats, knowledge_base):
    """Parse Service registry keys and return WindowsServiceInformation."""
    _ = knowledge_base
    services = {}
    field_map = {
        "Description": "description",
        "DisplayName": "display_name",
        "Group": "group_name",
        "DriverPackageId": "driver_package_id",
        "ErrorControl": "error_control",
        "ImagePath": "image_path",
        "ObjectName": "object_name",
        "Start": "startup_type",
        "Type": "service_type",
        "Parameters/ServiceDLL": "service_dll"
    }

    # Field map key should be converted to lowercase because key aquired through
    # self._GetKeyName could have some  characters in different case than the
    # field map, e.g. ServiceDLL and ServiceDll.
    field_map = {k.lower(): v for k, v in field_map.items()}
    for stat in stats:

      # Ignore subkeys
      if not stat.HasField("registry_data"):
        continue

      service_name = self._GetServiceName(stat.pathspec.path)
      reg_key = os.path.dirname(stat.pathspec.path)
      service_info = rdf_client.WindowsServiceInformation(
          name=service_name, registry_key=reg_key)
      services.setdefault(service_name, service_info)

      key = self._GetKeyName(stat.pathspec.path)

      if key in field_map:
        try:
          services[service_name].Set(field_map[key],
                                     stat.registry_data.GetValue())
        except type_info.TypeValueError:

          # Flatten multi strings into a simple string
          if (stat.registry_type ==
              rdf_client.StatEntry.RegistryType.REG_MULTI_SZ):
            services[service_name].Set(
                field_map[key],
                utils.SmartUnicode(stat.registry_data.GetValue()))
          else:
            # Log failures for everything else
            # TODO(user): change this to yield a ParserAnomaly object.
            dest_type = type(services[service_name].Get(field_map[key]))
            logging.debug("Wrong type set for %s:%s, expected %s, got %s",
                          stat.pathspec.path, stat.registry_data.GetValue(),
                          dest_type, type(stat.registry_data.GetValue()))

    return services.itervalues()
Beispiel #27
0
  def InsertArg(self, string="", **_):
    """Insert an arg to the current expression."""
    if self.verbose:
      logging.debug("Storing Argument %s", utils.SmartUnicode(string))

    # This expression is complete
    if self.current_expression.AddArg(string):
      self.stack.append(self.current_expression)
      self.current_expression = self.expression_cls()
      return self.PopState()
Beispiel #28
0
    def Publish(self, event_name, message):
        """Publish a message to an event queue.

    Args:
       event_name: The name of the event to publish to.
       message: An RDFValue instance to publish to the event listeners.
    """
        logging.debug("Publishing %s to %s",
                      utils.SmartUnicode(message)[:100], event_name)
        events.Events.PublishEvent(event_name, message, token=self.token)
Beispiel #29
0
 def _Decode(self, attribute, value):
   required_type = self._attribute_types.get(attribute, "bytes")
   if isinstance(value, buffer):
     value = str(value)
   if required_type in ("integer", "unsigned_integer"):
     return int(value)
   elif required_type == "string":
     return utils.SmartUnicode(value)
   else:
     return value
Beispiel #30
0
    def ResolveMulti(self, subject, attributes, timestamp=None, limit=None):
        subject = utils.SmartUnicode(subject)

        # Does timestamp represent a range?
        if isinstance(timestamp, (list, tuple)):
            start, end = timestamp  # pylint: disable=unpacking-non-sequence
        else:
            start, end = -1, 1 << 65

        start = int(start)
        end = int(end)

        if isinstance(attributes, str):
            attributes = [attributes]

        try:
            record = self.subjects[subject]
        except KeyError:
            return

        # Holds all the attributes which matched. Keys are attribute names, values
        # are lists of timestamped data.
        results = {}
        for attribute in attributes:
            for attr, values in record.iteritems():
                if attr == attribute:
                    for value, ts in values:
                        results_list = results.setdefault(attribute, [])
                        # If we are always after the latest ts we clear older ones.
                        if (results_list and timestamp == self.NEWEST_TIMESTAMP
                                and results_list[0][1] < ts):
                            results_list = []
                            results[attribute] = results_list

                        # Timestamp outside the range, drop it.
                        elif ts < start or ts > end:
                            continue

                        results_list.append((attribute, ts, value))

        # Return the results in the same order they requested.
        remaining_limit = limit
        for attribute in attributes:
            # This returns triples of (attribute_name, timestamp, data). We want to
            # sort by timestamp.
            for _, ts, data in sorted(results.get(attribute, []),
                                      key=lambda x: x[1],
                                      reverse=True):
                if remaining_limit:
                    remaining_limit -= 1
                    if remaining_limit == 0:
                        yield (attribute, data, ts)
                        return

                yield (attribute, data, ts)