Esempio n. 1
0
  def testHuntIsStoppedIfTotalNetworkUsageIsTooHigh(self):
    client_ids = self.SetupClients(5)

    hunt_id = self._CreateHunt(
        client_rule_set=foreman_rules.ForemanClientRuleSet(),
        client_rate=0,
        total_network_bytes_limit=5,
        args=self.GetFileHuntArgs())

    def CheckState(hunt_state, network_bytes_sent):
      hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
      self.assertEqual(hunt_obj.hunt_state, hunt_state)
      hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_id)
      self.assertEqual(hunt_counters.total_network_bytes_sent,
                       network_bytes_sent)

    self._RunHunt(
        client_ids[:2],
        client_mock=hunt_test_lib.SampleHuntMock(network_bytes_sent=2))

    # 4 is lower than the total limit. The hunt should still be running.
    CheckState(rdf_hunt_objects.Hunt.HuntState.STARTED, 4)

    self._RunHunt(
        [client_ids[2]],
        client_mock=hunt_test_lib.SampleHuntMock(network_bytes_sent=1))

    # 5 is equal to the total limit. Total network bytes sent should
    # go over the limit in order for the hunt to be stopped.
    CheckState(rdf_hunt_objects.Hunt.HuntState.STARTED, 5)

    self._RunHunt(
        [client_ids[3]],
        client_mock=hunt_test_lib.SampleHuntMock(network_bytes_sent=1))

    # 6 is greater than the total limit. The hunt should be stopped now.
    CheckState(rdf_hunt_objects.Hunt.HuntState.STOPPED, 6)

    self._RunHunt([client_ids[4]],
                  client_mock=hunt_test_lib.SampleHuntMock(
                      network_bytes_sent=2, failrate=-1))

    self._CheckHuntStoppedNotification(
        "reached the total network bytes sent limit")
Esempio n. 2
0
    def testOutputPluginsMaintainGlobalState(self):
        plugin_descriptor = rdf_output_plugin.OutputPluginDescriptor(
            plugin_name="StatefulDummyHuntOutputPlugin")

        self.assertListEqual(hunt_test_lib.StatefulDummyHuntOutputPlugin.data,
                             [])

        _ = self._CreateAndRunHunt(
            num_clients=5,
            client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=self.GetFileHuntArgs(),
            output_plugins=[plugin_descriptor])

        # Output plugins should have been called 5 times, adding a number
        # to the "data" list on every call and incrementing it each time.
        self.assertListEqual(hunt_test_lib.StatefulDummyHuntOutputPlugin.data,
                             [0, 1, 2, 3, 4])
Esempio n. 3
0
  def testPausingAndRestartingDoesNotStartHuntTwiceOnTheSameClient(self):
    """This tests if the hunt completes when some clients hang or raise."""
    client_ids = self.SetupClients(10)

    client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
        foreman_rules.ForemanClientRule(
            rule_type=foreman_rules.ForemanClientRule.Type.REGEX,
            regex=foreman_rules.ForemanRegexClientRule(
                field="CLIENT_NAME", attribute_regex="GRR"))
    ])

    with implementation.StartHunt(
        hunt_name=standard.SampleHunt.__name__,
        client_rule_set=client_rule_set,
        client_rate=0,
        token=self.token) as hunt:

      hunt.GetRunner().Start()

      hunt_id = hunt.urn

    foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
    for client_id in client_ids:
      num_tasks = foreman.AssignTasksToClient(client_id.Basename())
      self.assertEqual(num_tasks, 1)

    client_mock = hunt_test_lib.SampleHuntMock()
    hunt_test_lib.TestHuntHelper(client_mock, client_ids, False, self.token)

    # Pausing and running hunt: this leads to the fresh rules being written
    # to Foreman.RULES.
    with aff4.FACTORY.Open(hunt_id, mode="rw", token=self.token) as hunt:
      runner = hunt.GetRunner()
      runner.Pause()
      runner.Start()

    # Recreating the foreman so that it updates list of rules.
    foreman = aff4.FACTORY.Open("aff4:/foreman", mode="rw", token=self.token)
    for client_id in client_ids:
      num_tasks = foreman.AssignTasksToClient(client_id.Basename())
      # No tasks should be assigned as this hunt ran on all the clients
      # before.
      self.assertEqual(num_tasks, 0)
Esempio n. 4
0
    def testPausingAndRestartingDoesNotStartHuntTwiceOnTheSameClient(self):
        hunt_id, client_ids = self._CreateAndRunHunt(
            num_clients=10,
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=self.GetFileHuntArgs())

        for client_id in client_ids:
            flows = data_store.REL_DB.ReadAllFlowObjects(client_id=client_id)
            self.assertLen(flows, 1)

        hunt.PauseHunt(hunt_id)
        hunt.StartHunt(hunt_id)

        self._RunHunt(client_ids)

        for client_id in client_ids:
            flows = data_store.REL_DB.ReadAllFlowObjects(client_id=client_id)
            self.assertLen(flows, 1)
    def testFromDictMixedRules(self):
        dct = {
            "match_mode":
            "MATCH_ANY",
            "rules": [
                {
                    "rule_type": "OS",
                    "os": {
                        "os_windows": True,
                        "os_linux": True,
                    },
                },
                {
                    "rule_type": "LABEL",
                    "label": {
                        "label_names": ["foo", "bar"],
                        "match_mode": "MATCH_ALL",
                    },
                },
            ],
        }

        rdf = foreman_rules.ForemanClientRuleSet()
        rdf.FromDict(dct)

        self.assertEqual(
            rdf.match_mode,
            foreman_rules.ForemanClientRuleSet.MatchMode.MATCH_ANY)
        self.assertLen(rdf.rules, 2)

        self.assertEqual(rdf.rules[0].rule_type,
                         foreman_rules.ForemanClientRule.Type.OS)
        self.assertTrue(rdf.rules[0].os.os_windows)
        self.assertTrue(rdf.rules[0].os.os_linux)
        self.assertFalse(rdf.rules[0].os.os_darwin)

        self.assertEqual(rdf.rules[1].rule_type,
                         foreman_rules.ForemanClientRule.Type.LABEL)
        self.assertEqual(rdf.rules[1].label.label_names, ["foo", "bar"])
        self.assertEqual(
            rdf.rules[1].label.match_mode,
            foreman_rules.ForemanLabelClientRule.MatchMode.MATCH_ALL)
Esempio n. 6
0
    def testBrokenHunt(self):
        """This tests the behavior when a hunt raises an exception."""

        # Set up 10 clients.
        client_ids = self.SetupClients(10)

        client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.REGEX,
                regex=foreman_rules.ForemanRegexClientRule(
                    field="CLIENT_NAME", attribute_regex="GRR"))
        ])

        with implementation.StartHunt(hunt_name=BrokenSampleHunt.__name__,
                                      client_rule_set=client_rule_set,
                                      client_rate=0,
                                      token=self.token) as hunt:

            hunt.GetRunner().Start()

        foreman = aff4.FACTORY.Open("aff4:/foreman",
                                    mode="rw",
                                    token=self.token)
        for client_id in client_ids:
            foreman.AssignTasksToClient(client_id.Basename())

        # Run the hunt.
        client_mock = hunt_test_lib.SampleHuntMock(failrate=2)
        hunt_test_lib.TestHuntHelper(client_mock, client_ids, False,
                                     self.token)

        hunt_obj = aff4.FACTORY.Open(hunt.session_id,
                                     mode="rw",
                                     age=aff4.ALL_TIMES,
                                     token=self.token)
        started, finished, errors = hunt_obj.GetClientsCounts()

        self.assertEqual(started, 10)
        # There should be errors for the five clients where the hunt raised.
        self.assertEqual(errors, 5)
        # All of the clients that have the file should still finish eventually.
        self.assertEqual(finished, 5)
Esempio n. 7
0
    def testUpdatesStatsCounterOnOutputPluginFailure(self):
        plugin_descriptor = rdf_output_plugin.OutputPluginDescriptor(
            plugin_name="FailingDummyHuntOutputPlugin")

        # 1 error for each client makes it 5 errors, 0 results.
        with self.assertStatsCounterDelta(
                0,
                flow_base.HUNT_RESULTS_RAN_THROUGH_PLUGIN,
                fields=["FailingDummyHuntOutputPlugin"]):
            with self.assertStatsCounterDelta(
                    5,
                    flow_base.HUNT_OUTPUT_PLUGIN_ERRORS,
                    fields=["FailingDummyHuntOutputPlugin"]):
                self._CreateAndRunHunt(
                    num_clients=5,
                    client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
                    client_rule_set=foreman_rules.ForemanClientRuleSet(),
                    client_rate=0,
                    args=self.GetFileHuntArgs(),
                    output_plugins=[plugin_descriptor])
Esempio n. 8
0
  def testHuntClientRateIsAppliedCorrectly(self):
    now = rdfvalue.RDFDatetime.Now()

    _, client_ids = self._CreateAndRunHunt(
        num_clients=10,
        client_rule_set=foreman_rules.ForemanClientRuleSet(),
        client_rate=1,
        args=self.GetFileHuntArgs())

    requests = data_store.REL_DB.ReadFlowProcessingRequests()
    requests.sort(key=lambda r: r.delivery_time)

    # The first request is scheduled to run immediately and has been processed
    # already.
    self.assertLen(requests, 9)
    for i, (r, client_id) in enumerate(zip(requests, client_ids[1:])):
      self.assertEqual(r.client_id, client_id)
      time_diff = r.delivery_time - (
          now + rdfvalue.Duration.From(1, rdfvalue.MINUTES) * (i + 1))
      self.assertLess(time_diff, rdfvalue.Duration.From(5, rdfvalue.SECONDS))
Esempio n. 9
0
  def testEvaluatesPositiveInMatchAllModeIfAllRuleMatch(self):
    # Instantiate a rule set that matches if all of its two
    # operating system rules match
    rs = foreman_rules.ForemanClientRuleSet(
        match_mode=foreman_rules.ForemanClientRuleSet.MatchMode.MATCH_ALL,
        rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.OS,
                os=foreman_rules.ForemanOsClientRule(
                    os_windows=False, os_linux=True, os_darwin=False)),
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.OS,
                os=foreman_rules.ForemanOsClientRule(
                    os_windows=False, os_linux=True, os_darwin=True))
        ])

    client_id_lin = self.SetupClient(0, system="Linux")
    # All of the set's rules have os_linux=False, so the whole set matches
    self.assertTrue(
        rs.Evaluate(data_store.REL_DB.ReadClientFullInfo(client_id_lin)))
Esempio n. 10
0
    def testFailingOutputPluginDoesNotAffectOtherOutputPlugins(self):
        failing_plugin_descriptor = rdf_output_plugin.OutputPluginDescriptor(
            plugin_name="FailingDummyHuntOutputPlugin")
        plugin_descriptor = rdf_output_plugin.OutputPluginDescriptor(
            plugin_name="DummyHuntOutputPlugin")

        hunt_id, _ = self._CreateAndRunHunt(
            num_clients=5,
            client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=self.GetFileHuntArgs(),
            output_plugins=[failing_plugin_descriptor, plugin_descriptor])

        errors = hunt.GetHuntOutputPluginErrors(hunt_id, 0, sys.maxsize)
        self.assertLen(errors, 5)

        # Check that non-failing output plugin is still correctly processed.
        logs = hunt.GetHuntOutputPluginLogs(hunt_id, 0, sys.maxsize)
        self.assertLen(logs, 5)
Esempio n. 11
0
  def testEvaluatesNegativeInMatchAllModeIfOnlyOneRuleMatches(self):
    # Instantiate a rule set that matches if all of its two
    # operating system rules match
    rs = foreman_rules.ForemanClientRuleSet(
        match_mode=foreman_rules.ForemanClientRuleSet.MatchMode.MATCH_ALL,
        rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.OS,
                os=foreman_rules.ForemanOsClientRule(
                    os_windows=False, os_linux=True, os_darwin=False)),
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.OS,
                os=foreman_rules.ForemanOsClientRule(
                    os_windows=False, os_linux=True, os_darwin=True))
        ])

    client_id_dar = self.SetupClient(0, system="Darwin")
    # One of the set's rules has os_darwin=False, so the whole set doesn't
    # match with the match all match mode
    self.assertFalse(
        rs.Evaluate(data_store.REL_DB.ReadClientFullInfo(client_id_dar)))
Esempio n. 12
0
  def testEvaluatesPositiveInMatchAnyModeIfOneRuleMatches(self):
    # Instantiate a rule set that matches if any of its two
    # operating system rules matches
    rs = foreman_rules.ForemanClientRuleSet(
        match_mode=foreman_rules.ForemanClientRuleSet.MatchMode.MATCH_ANY,
        rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.OS,
                os=foreman_rules.ForemanOsClientRule(
                    os_windows=False, os_linux=True, os_darwin=False)),
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.OS,
                os=foreman_rules.ForemanOsClientRule(
                    os_windows=False, os_linux=True, os_darwin=True))
        ])

    client_id_dar = self.SetupClient(0, system="Darwin")
    # One of the set's rules has os_darwin=True, so the whole set matches
    # with the match any match mode
    self.assertTrue(
        rs.Evaluate(aff4.FACTORY.Open(client_id_dar, token=self.token)))
Esempio n. 13
0
    def testOutputPluginsErrorsAreCorrectlyWrittenAndCanBeRead(self):
        failing_plugin_descriptor = rdf_output_plugin.OutputPluginDescriptor(
            plugin_name="FailingDummyHuntOutputPlugin")

        hunt_id, _ = self._CreateAndRunHunt(
            num_clients=5,
            client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=self.GetFileHuntArgs(),
            output_plugins=[failing_plugin_descriptor])

        errors = hunt.GetHuntOutputPluginErrors(hunt_id, 0, sys.maxsize)
        self.assertLen(errors, 5)
        for e in errors:
            self.assertEqual(e.batch_size, 1)
            self.assertEqual(
                e.status,
                output_plugin.OutputPluginBatchProcessingStatus.Status.ERROR)
            self.assertEqual(e.plugin_descriptor, failing_plugin_descriptor)
            self.assertEqual(e.summary, "Oh no!")
Esempio n. 14
0
    def testResultsAreCorrectlyCounted(self):
        path = os.path.join(self.base_path, "*hello*")
        num_files = len(glob.glob(path))
        self.assertGreater(num_files, 1)

        hunt_args = rdf_hunt_objects.HuntArguments.Standard(
            flow_name=compatibility.GetName(file_finder.FileFinder),
            flow_args=rdf_file_finder.FileFinderArgs(
                paths=[path],
                action=rdf_file_finder.FileFinderAction(action_type="STAT"),
            ))
        hunt_id, _ = self._CreateAndRunHunt(
            num_clients=5,
            client_mock=action_mocks.FileFinderClientMock(),
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=hunt_args)

        hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_id)
        self.assertEqual(hunt_counters.num_clients_with_results, 5)
        self.assertEqual(hunt_counters.num_results, 5 * num_files)
Esempio n. 15
0
    def testHangingClientsAreCorrectlyAccountedFor(self):
        client_ids = self.SetupClients(10)

        hunt_obj = rdf_hunt_objects.Hunt(
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=self.GetFileHuntArgs())
        hunt.CreateHunt(hunt_obj)
        hunt_obj = hunt.StartHunt(hunt_obj.hunt_id)

        foreman_obj = foreman.Foreman()
        for client_id in client_ids:
            foreman_obj.AssignTasksToClient(client_id)

        client_mock = hunt_test_lib.SampleHuntMock(failrate=2)
        hunt_test_lib.TestHuntHelper(client_mock, client_ids[1:9])

        hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_obj.hunt_id)
        self.assertEqual(hunt_counters.num_clients, 10)
        self.assertEqual(hunt_counters.num_successful_clients, 4)
        self.assertEqual(hunt_counters.num_failed_clients, 4)
Esempio n. 16
0
  def testEvaluatesNegativeInMatchAnyModeIfNoRuleMatches(self):
    # Instantiate a rule set that matches if any of its two
    # operating system rules matches
    rs = foreman_rules.ForemanClientRuleSet(
        match_mode=foreman_rules.ForemanClientRuleSet.MatchMode.MATCH_ANY,
        rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.OS,
                os=foreman_rules.ForemanOsClientRule(
                    os_windows=False, os_linux=True, os_darwin=False)),
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.OS,
                os=foreman_rules.ForemanOsClientRule(
                    os_windows=False, os_linux=True, os_darwin=True))
        ])

    client_id_win = self.SetupClient(0, system="Windows")
    # None of the set's rules has os_windows=True, so the whole set doesn't
    # match
    self.assertFalse(
        rs.Evaluate(data_store.REL_DB.ReadClientFullInfo(client_id_win)))
Esempio n. 17
0
    def _RunRateLimitedHunt(self, client_ids, start_time):
        client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.REGEX,
                regex=foreman_rules.ForemanRegexClientRule(
                    field="CLIENT_NAME", attribute_regex="GRR"))
        ])

        with implementation.StartHunt(hunt_name=DummyHunt.__name__,
                                      client_rule_set=client_rule_set,
                                      client_rate=1,
                                      token=self.token) as hunt:
            hunt.Run()

        # Pretend to be the foreman now and dish out hunting jobs to all the
        # clients..
        foreman = aff4.FACTORY.Open("aff4:/foreman",
                                    mode="rw",
                                    token=self.token)
        for client_id in client_ids:
            foreman.AssignTasksToClient(client_id.Basename())

        self.assertEmpty(DummyHunt.client_ids)

        # Run the hunt.
        worker_mock = worker_test_lib.MockWorker(check_flow_errors=True,
                                                 queues=[queues.HUNTS],
                                                 token=self.token)

        # One client is scheduled in the first minute.
        with test_lib.FakeTime(start_time + 2):
            worker_mock.Simulate()
        self.assertLen(DummyHunt.client_ids, 1)

        # No further clients will be scheduled until the end of the first minute.
        with test_lib.FakeTime(start_time + 59):
            worker_mock.Simulate()
        self.assertLen(DummyHunt.client_ids, 1)

        return worker_mock, hunt.urn
Esempio n. 18
0
    def testClientLimit(self):
        """This tests that we can limit hunts to a number of clients."""

        # Set up 10 clients.
        client_ids = self.SetupClients(10)

        client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.REGEX,
                regex=foreman_rules.ForemanRegexClientRule(
                    field="CLIENT_NAME", attribute_regex="GRR"))
        ])

        with implementation.StartHunt(hunt_name=standard.SampleHunt.__name__,
                                      client_limit=5,
                                      client_rule_set=client_rule_set,
                                      client_rate=0,
                                      token=self.token) as hunt:
            hunt.Run()

        foreman = aff4.FACTORY.Open("aff4:/foreman",
                                    mode="rw",
                                    token=self.token)
        for client_id in client_ids:
            foreman.AssignTasksToClient(client_id.Basename())

        # Run the hunt.
        client_mock = hunt_test_lib.SampleHuntMock(failrate=2)
        hunt_test_lib.TestHuntHelper(client_mock, client_ids, False,
                                     self.token)

        hunt_obj = aff4.FACTORY.Open(hunt.urn,
                                     mode="rw",
                                     age=aff4.ALL_TIMES,
                                     token=self.token)

        started, finished, _ = hunt_obj.GetClientsCounts()
        # We limited here to 5 clients.
        self.assertEqual(started, 5)
        self.assertEqual(finished, 5)
Esempio n. 19
0
    def testHangingClients(self):
        """This tests if the hunt completes when some clients hang or raise."""
        # Set up 10 clients.
        client_ids = self.SetupClients(10)

        client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.REGEX,
                regex=foreman_rules.ForemanRegexClientRule(
                    field="CLIENT_NAME", attribute_regex="GRR"))
        ])

        with implementation.StartHunt(hunt_name=standard.SampleHunt.__name__,
                                      client_rule_set=client_rule_set,
                                      client_rate=0,
                                      token=self.token) as hunt:

            hunt.GetRunner().Start()

        foreman = aff4.FACTORY.Open("aff4:/foreman",
                                    mode="rw",
                                    token=self.token)
        for client_id in client_ids:
            foreman.AssignTasksToClient(client_id.Basename())

        client_mock = hunt_test_lib.SampleHuntMock(failrate=2)
        # Just pass 8 clients to run, the other two went offline.
        hunt_test_lib.TestHuntHelper(client_mock, client_ids[1:9], False,
                                     self.token)

        hunt_obj = aff4.FACTORY.Open(hunt.session_id,
                                     mode="rw",
                                     age=aff4.ALL_TIMES,
                                     token=self.token)

        started, finished, _ = hunt_obj.GetClientsCounts()
        # We started the hunt on 10 clients.
        self.assertEqual(started, 10)
        # But only 8 should have finished.
        self.assertEqual(finished, 8)
Esempio n. 20
0
    def testProcessing(self):
        """This tests running the hunt on some clients."""

        # Set up 10 clients.
        client_ids = self.SetupClients(10)

        client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.REGEX,
                regex=foreman_rules.ForemanRegexClientRule(
                    field="CLIENT_NAME", attribute_regex="GRR"))
        ])

        with implementation.StartHunt(hunt_name=standard.SampleHunt.__name__,
                                      client_rule_set=client_rule_set,
                                      client_rate=0,
                                      token=self.token) as hunt:

            hunt.GetRunner().Start()

        foreman = aff4.FACTORY.Open("aff4:/foreman",
                                    mode="rw",
                                    token=self.token)
        for client_id in client_ids:
            foreman.AssignTasksToClient(client_id.Basename())

        # Run the hunt.
        client_mock = hunt_test_lib.SampleHuntMock(failrate=2)
        hunt_test_lib.TestHuntHelper(client_mock, client_ids, False,
                                     self.token)

        hunt_obj = aff4.FACTORY.Open(hunt.session_id,
                                     mode="r",
                                     age=aff4.ALL_TIMES,
                                     aff4_type=standard.SampleHunt,
                                     token=self.token)

        started, finished, _ = hunt_obj.GetClientsCounts()
        self.assertEqual(started, 10)
        self.assertEqual(finished, 10)
Esempio n. 21
0
    def testCopyHuntPreservesRuleType(self):
        implementation.StartHunt(
            hunt_name=standard.GenericHunt.__name__,
            description="model hunt",
            flow_runner_args=rdf_flow_runner.FlowRunnerArgs(
                flow_name=transfer.GetFile.__name__),
            flow_args=transfer.GetFileArgs(pathspec=rdf_paths.PathSpec(
                path="/tmp/evil.txt",
                pathtype=rdf_paths.PathSpec.PathType.TSK,
            )),
            client_rule_set=foreman_rules.ForemanClientRuleSet(rules=[
                foreman_rules.ForemanClientRule(
                    rule_type=foreman_rules.ForemanClientRule.Type.OS,
                    os=foreman_rules.ForemanOsClientRule(os_darwin=True))
            ]),
            token=self.token)

        self.Open("/#main=ManageHunts")
        self.Click("css=tr:contains('model hunt')")
        self.Click("css=button[name=CopyHunt]:not([disabled])")

        # Wait until dialog appears.
        self.WaitUntil(self.IsElementPresent,
                       "css=grr-wizard-form:contains('What to run?')")
        # Click on "Next" button
        self.Click("css=grr-new-hunt-wizard-form button.Next")
        self.WaitUntil(self.IsElementPresent,
                       "css=grr-wizard-form:contains('Hunt parameters')")
        # Click on "Next" button.
        self.Click("css=grr-new-hunt-wizard-form button.Next")
        self.WaitUntil(
            self.IsElementPresent,
            "css=grr-wizard-form:contains('How to process results')")
        # Click on "Next" button
        self.Click("css=grr-new-hunt-wizard-form button.Next")
        self.WaitUntil(self.IsElementPresent,
                       "css=grr-wizard-form:contains('Where to run?')")
        self.WaitUntil(
            self.IsElementPresent, "css=grr-new-hunt-wizard-form "
            "label:contains('Os darwin') ~ * input:checked")
Esempio n. 22
0
    def testHuntFlowLogsAreCorrectlyWrittenAndCanBeRead(self):
        hunt_args = rdf_hunt_objects.HuntArguments.Standard(
            flow_name=compatibility.GetName(flow_test_lib.DummyLogFlow))
        hunt_id, client_ids = self._CreateAndRunHunt(
            num_clients=10,
            client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=hunt_args)

        hunt_logs = data_store.REL_DB.ReadHuntLogEntries(
            hunt_id, 0, sys.maxsize)
        # 4 logs for each flow, 2 flow run per client.
        self.assertLen(hunt_logs, 8 * len(client_ids))
        self.assertCountEqual(set(log.client_id for log in hunt_logs),
                              [c.Basename() for c in client_ids])
        self.assertCountEqual(set(log.message for log in hunt_logs), [
            "First", "Second", "Third", "Fourth", "Uno", "Dos", "Tres",
            "Cuatro"
        ])

        for log in hunt_logs:
            self.assertEqual(log.hunt_id, hunt_id)
Esempio n. 23
0
    def testHuntIsStoppedIfCrashNumberOverThreshold(self):
        client_ids = self.SetupClients(4)

        hunt_id = self._CreateHunt(
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            crash_limit=3,
            args=self.GetFileHuntArgs())

        client_mock = flow_test_lib.CrashClientMock()
        self._RunHunt(client_ids[:2], client_mock=client_mock)

        hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
        self.assertEqual(hunt_obj.hunt_state,
                         rdf_hunt_objects.Hunt.HuntState.STARTED)

        self._RunHunt(client_ids[2:], client_mock=client_mock)

        hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
        self.assertEqual(hunt_obj.hunt_state,
                         rdf_hunt_objects.Hunt.HuntState.STOPPED)

        self._CheckHuntStoppedNotification("reached the crashes limit")
Esempio n. 24
0
  def testPausingTheHuntChangingParametersAndStartingAgainWorksAsExpected(self):
    client_ids = self.SetupClients(2)

    hunt_id = self._CreateHunt(
        client_rule_set=foreman_rules.ForemanClientRuleSet(),
        client_rate=0,
        client_limit=1,
        args=self.GetFileHuntArgs())

    self._RunHunt(client_ids[:2])
    hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
    self.assertEqual(hunt_obj.hunt_state,
                     rdf_hunt_objects.Hunt.HuntState.PAUSED)
    # There should be only one client, due to the limit
    hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_id)
    self.assertEqual(hunt_counters.num_clients, 1)

    hunt.UpdateHunt(hunt_id, client_limit=10)
    hunt.StartHunt(hunt_id)

    self._RunHunt(client_ids[:2])
    hunt_counters = data_store.REL_DB.ReadHuntCounters(hunt_id)
    self.assertEqual(hunt_counters.num_clients, 2)
Esempio n. 25
0
    def testResourceUsageStatsAreReportedCorrectly(self):
        hunt_id, _ = self._CreateAndRunHunt(
            num_clients=10,
            client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=self.GetFileHuntArgs())

        hunt_obj = data_store.REL_DB.ReadHuntObject(hunt_id)
        usage_stats = hunt_obj.client_resources_stats

        # Values below are calculated based on SampleHuntMock's behavior.
        self.assertEqual(usage_stats.user_cpu_stats.num, 10)
        self.assertAlmostEqual(usage_stats.user_cpu_stats.mean, 5.5)
        self.assertAlmostEqual(usage_stats.user_cpu_stats.std, 2.8722813)

        self.assertEqual(usage_stats.system_cpu_stats.num, 10)
        self.assertAlmostEqual(usage_stats.system_cpu_stats.mean, 11)
        self.assertAlmostEqual(usage_stats.system_cpu_stats.std, 5.7445626)

        self.assertEqual(usage_stats.network_bytes_sent_stats.num, 10)
        self.assertAlmostEqual(usage_stats.network_bytes_sent_stats.mean, 16.5)
        self.assertAlmostEqual(usage_stats.network_bytes_sent_stats.std,
                               8.61684396)

        # NOTE: Not checking histograms here. RunningStatsTest tests that mean,
        # standard deviation and histograms are calculated correctly. Therefore
        # if mean/stdev values are correct histograms should be ok as well.

        self.assertLen(usage_stats.worst_performers, 10)

        prev = usage_stats.worst_performers[0]
        for p in usage_stats.worst_performers[1:]:
            self.assertGreater(
                prev.cpu_usage.user_cpu_time + prev.cpu_usage.system_cpu_time,
                p.cpu_usage.user_cpu_time + p.cpu_usage.system_cpu_time)
            prev = p
Esempio n. 26
0
    def testForemanRulesAreCorrectlyPropagatedWhenHuntStarts(self):
        client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.REGEX,
                regex=foreman_rules.ForemanRegexClientRule(
                    field="CLIENT_NAME", attribute_regex="HUNT")),
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.INTEGER,
                integer=foreman_rules.ForemanIntegerClientRule(
                    field="CLIENT_CLOCK",
                    operator=foreman_rules.ForemanIntegerClientRule.Operator.
                    GREATER_THAN,
                    value=1336650631137737))
        ])

        self.assertEmpty(data_store.REL_DB.ReadAllForemanRules())

        hunt_obj = rdf_hunt_objects.Hunt(client_rule_set=client_rule_set)
        hunt_obj.args.hunt_type = hunt_obj.args.HuntType.STANDARD
        data_store.REL_DB.WriteHuntObject(hunt_obj)

        hunt_obj = hunt.StartHunt(hunt_obj.hunt_id)

        rules = data_store.REL_DB.ReadAllForemanRules()
        self.assertLen(rules, 1)
        rule = rules[0]
        self.assertEqual(rule.client_rule_set, client_rule_set)
        self.assertEqual(rule.hunt_id, hunt_obj.hunt_id)
        self.assertEqual(rule.expiration_time,
                         hunt_obj.init_start_time + hunt_obj.duration)

        # Running a second time should not change the rules any more.
        with self.assertRaises(hunt.OnlyPausedHuntCanBeStartedError):
            hunt.StartHunt(hunt_obj.hunt_id)
        rules = data_store.REL_DB.ReadAllForemanRules()
        self.assertLen(rules, 1)
Esempio n. 27
0
  def testHuntFlowLogsAreCorrectlyWrittenAndCanBeRead(self):
    hunt_args = rdf_hunt_objects.HuntArguments.Standard(
        flow_name=compatibility.GetName(flow_test_lib.DummyLogFlow))
    hunt_id, client_ids = self._CreateAndRunHunt(
        num_clients=10,
        client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
        client_rule_set=foreman_rules.ForemanClientRuleSet(),
        client_rate=0,
        args=hunt_args)

    hunt_logs = data_store.REL_DB.ReadHuntLogEntries(hunt_id, 0, sys.maxsize)
    # 4 logs for each flow. Note: DummyLogFlow also calls DummyLogFlowChild,
    # but children flows logs should not be present in the output.
    self.assertLen(hunt_logs, 4 * len(client_ids))
    self.assertCountEqual(set(log.client_id for log in hunt_logs), client_ids)

    messages_set = set(log.message for log in hunt_logs)
    self.assertCountEqual(messages_set, ["First", "Second", "Third", "Fourth"])

    for nested_flow_log in ["Uno", "Dos", "Tres", "Cuatro"]:
      self.assertNotIn(nested_flow_log, messages_set)

    for log in hunt_logs:
      self.assertEqual(log.hunt_id, hunt_id)
Esempio n. 28
0
    def testOutputPluginFlushErrorIsLoggedProperly(self):
        plugin_descriptor = rdf_output_plugin.OutputPluginDescriptor(
            plugin_name="FailingInFlushDummyHuntOutputPlugin")

        hunt_id, _ = self._CreateAndRunHunt(
            num_clients=5,
            client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=self.GetFileHuntArgs(),
            output_plugins=[plugin_descriptor])

        logs = hunt.GetHuntOutputPluginLogs(hunt_id, 0, sys.maxsize)
        self.assertEmpty(logs)

        errors = hunt.GetHuntOutputPluginErrors(hunt_id, 0, sys.maxsize)
        self.assertLen(errors, 5)
        for e in errors:
            self.assertEqual(e.batch_size, 1)
            self.assertEqual(
                e.status,
                output_plugin.OutputPluginBatchProcessingStatus.Status.ERROR)
            self.assertEqual(e.plugin_descriptor, plugin_descriptor)
            self.assertEqual(e.summary, "Flush, oh no!")
Esempio n. 29
0
    def testOutputPluginFlushErrorIsLoggedProperly(self):
        plugin_descriptor = rdf_output_plugin.OutputPluginDescriptor(
            plugin_name="FailingInFlushDummyHuntOutputPlugin")

        hunt_id, client_ids = self._CreateAndRunHunt(
            num_clients=5,
            client_mock=hunt_test_lib.SampleHuntMock(failrate=-1),
            client_rule_set=foreman_rules.ForemanClientRuleSet(),
            client_rate=0,
            args=self.GetFileHuntArgs(),
            output_plugins=[plugin_descriptor])

        logs = data_store.REL_DB.ReadHuntOutputPluginLogEntries(
            hunt_id,
            output_plugin_id="0",
            offset=0,
            count=sys.maxsize,
            with_type=rdf_flow_objects.FlowOutputPluginLogEntry.LogEntryType.
            LOG)
        self.assertEmpty(logs)

        errors = data_store.REL_DB.ReadHuntOutputPluginLogEntries(
            hunt_id,
            output_plugin_id="0",
            offset=0,
            count=sys.maxsize,
            with_type=rdf_flow_objects.FlowOutputPluginLogEntry.LogEntryType.
            ERROR)
        self.assertLen(errors, 5)
        self.assertItemsEqual([e.client_id for e in errors],
                              [cid.Basename() for cid in client_ids])
        for e in errors:
            self.assertEqual(e.hunt_id, hunt_id)
            self.assertGreater(e.timestamp, 0)
            self.assertEqual(
                e.message, "Error while processing 1 replies: Flush, oh no!")
Esempio n. 30
0
    def testCallback(self, client_limit=None):
        """Checks that the foreman uses the callback specified in the action."""
        client_urn = self.SetupClient(0)
        client_rule_set = foreman_rules.ForemanClientRuleSet(rules=[
            foreman_rules.ForemanClientRule(
                rule_type=foreman_rules.ForemanClientRule.Type.REGEX,
                regex=foreman_rules.ForemanRegexClientRule(
                    field="CLIENT_NAME", attribute_regex="GRR"))
        ])

        with implementation.StartHunt(hunt_name=standard.SampleHunt.__name__,
                                      client_rule_set=client_rule_set,
                                      client_limit=client_limit,
                                      client_rate=0,
                                      token=self.token) as hunt:

            hunt.GetRunner().Start()

        # Create a client that matches our regex.
        with aff4.FACTORY.Open(client_urn, mode="rw",
                               token=self.token) as client:
            info = client.Schema.CLIENT_INFO()
            info.client_name = "GRR Monitor"
            client.Set(client.Schema.CLIENT_INFO, info)

        foreman = aff4.FACTORY.Open("aff4:/foreman",
                                    mode="rw",
                                    token=self.token)
        with utils.Stubber(standard.SampleHunt, "StartClients", self.Callback):
            self.called = []

            client_id = client_urn.Basename()
            foreman.AssignTasksToClient(client_id)

            self.assertLen(self.called, 1)
            self.assertEqual(self.called[0][1], [client_id])