Example #1
0
 def testEq(self):
     """Test value_eq on LazyLineReader."""
     baseline = repeated.meld("Alice\n", "Bob\n", "Charlie\n", "Dave\n",
                              "Eve\n", "Frank")
     with open(testlib.get_fixture_path("names.txt"), "r") as fd:
         reader = line_reader.LazyLineReader(fd)
         self.assertValuesEqual(baseline, reader)
Example #2
0
 def testReading(self):
     """Test that reader reads the same lines as the File object."""
     with open(testlib.get_fixture_path("names.txt"), "r") as fd:
         line_count = len(list(fd))
         # The fact that the fd.tell() is now at EOF shouldn't matter.
         reader = line_reader.LazyLineReader(fd)
         self.assertEqual(len(list(reader)), line_count)
Example #3
0
    def testGroup(self):
        result = api.apply(
            query=q.Query(
                (
                    "group",
                    # The input:
                    ("apply", ("var", "csv"), ("param", 0), True),
                    # The grouper expression:
                    ("var", "country"),

                    # The output reducers:
                    ("reducer", ("var", "singleton"), ("var", "country")),
                    ("reducer", ("var", "mean"), ("cast", ("var", "age"),
                                                  ("var", "int"))),
                    ("reducer", ("var", "sum"), ("cast", ("var", "age"),
                                                 ("var", "int")))),
                params=[testlib.get_fixture_path("fake_users.csv")]),
            allow_io=True)

        # Round the output means for comparison.
        actual = []
        for row in result:
            row[1] = int(row[1])
            actual.append(row)

        expected = repeated.meld(['El Salvador', 55, 1287],
                                 ['Ethiopia', 55, 1210],
                                 ['French Guiana', 47, 381],
                                 ['Germany', 42, 299], ['Haiti', 46, 610],
                                 ['Mayotte', 50, 865], ['Portugal', 48, 485])

        self.assertItemsEqual(expected, actual)
Example #4
0
    def testFullRun(self):
        tagfile = tag.TagFile(
            path=testlib.get_fixture_path("plaso_tagfile.txt"))

        # This is just a sanity check. Nuanced tests for the tagfile parser
        # are left as exercise to the reader.
        self.assertIsInstance(tagfile.root, ast.Repeat)
Example #5
0
 def testDicts(self):
     """Test building dicts."""
     with open(testlib.get_fixture_path("small.csv"), "r") as fd:
         reader = csv_reader.LazyCSVReader(fd, output_dicts=True)
         first_row = next(iter(reader))
         self.assertEqual(dict(Name="Alice", Age="25", City="Zurich"),
                          first_row)
Example #6
0
 def testReading(self):
     """Test that reader reads the same lines as the File object."""
     with open(testlib.get_fixture_path("names.txt"), "r") as fd:
         line_count = len(list(fd))
         # The fact that the fd.tell() is now at EOF shouldn't matter.
         reader = line_reader.LazyLineReader(fd)
         self.assertEqual(len(list(reader)), line_count)
Example #7
0
 def testEq(self):
     """Test value_eq on LazyLineReader."""
     baseline = repeated.meld("Alice\n", "Bob\n", "Charlie\n", "Dave\n",
                              "Eve\n", "Frank")
     with open(testlib.get_fixture_path("names.txt"), "r") as fd:
         reader = line_reader.LazyLineReader(fd)
         self.assertValuesEqual(baseline, reader)
Example #8
0
    def testFullRun(self):
        tagfile = tag.TagFile(
            path=testlib.get_fixture_path("plaso_tagfile.txt"))

        # This is just a sanity check. Nuanced tests for the tagfile parser
        # are left as exercise to the reader.
        self.assertIsInstance(tagfile.root, ast.Repeat)
Example #9
0
    def testRestarting(self):
        """Test that the reader can restart and support multiple users."""
        with open(testlib.get_fixture_path("names.txt"), "r") as fd:
            reader = line_reader.LazyLineReader(fd)
            iterator = iter(reader)
            iterator2 = iter(reader)

            self.assertEqual(next(iterator), next(iterator2))
Example #10
0
    def testRestarting(self):
        """Test that the reader can restart and support multiple users."""
        with open(testlib.get_fixture_path("names.txt"), "r") as fd:
            reader = line_reader.LazyLineReader(fd)
            iterator = iter(reader)
            iterator2 = iter(reader)

            self.assertEqual(next(iterator), next(iterator2))
Example #11
0
    def testCloseInDestructor(self):
        fd = open(testlib.get_fixture_path("names.txt"), "r")
        reader = csv_reader.LazyCSVReader(fd)

        for _ in reader:
            pass

        reader = None
        self.assertTrue(fd.closed)
Example #12
0
    def testCloseInDestructor(self):
        fd = open(testlib.get_fixture_path("names.txt"), "r")
        reader = line_reader.LazyLineReader(fd)

        for _ in reader:
            pass

        reader = None
        self.assertTrue(fd.closed)
Example #13
0
    def run(self):
        replacements = []
        if self.fixture_name is not None:
            replacements.append(testlib.get_fixture_path(self.fixture_name))

        result = api.apply(self.query, replacements=replacements, allow_io=True)

        # Force lazy results to be realized, but don't do anything with them.
        for _ in repeated.getvalues(result):
            pass
Example #14
0
    def testIOReads(self):
        """Test that IO is properly hooked up when requested."""
        self.assertValuesEqual(
            api.apply("SELECT * FROM csv(?)",
                      replacements=[testlib.get_fixture_path("small.csv")],
                      allow_io=True), repeated.meld(*small_csv.EXPECTED))

        # Keyword arguments to 'csv' should work.
        result = api.apply(
            "SELECT * FROM csv(?, decode_header: true)",
            replacements=[testlib.get_fixture_path("small.csv")],
            allow_io=True)

        first_row = next(iter(result))
        self.assertEqual(dict(Name="Alice", Age="25", City="Zurich"),
                         first_row)

        # The FD closes, right?
        fd = result.source.fd
        result = None
        self.assertTrue(fd.closed)
Example #15
0
    def testIOReads(self):
        """Test that IO is properly hooked up when requested."""
        self.assertValuesEqual(
            api.apply("SELECT * FROM csv(?)",
                      replacements=[testlib.get_fixture_path("small.csv")],
                      allow_io=True),
            repeated.meld(*small_csv.EXPECTED))

        # Keyword arguments to 'csv' should work.
        result = api.apply(
            "SELECT * FROM csv(?, decode_header: true)",
            replacements=[testlib.get_fixture_path("small.csv")],
            allow_io=True)

        first_row = next(iter(result))
        self.assertEqual(dict(Name="Alice", Age="25", City="Zurich"),
                         first_row)

        # The FD closes, right?
        fd = result.source.fd
        result = None
        self.assertTrue(fd.closed)
Example #16
0
    def testGroup(self):
        result = api.apply(
            query=q.Query(
                ("group",
                    # The input:
                    ("apply",
                        ("var", "csv"),
                        ("param", 0),
                        True),
                    # The grouper expression:
                    ("var", "country"),

                    # The output reducers:
                    ("reducer",
                        ("var", "singleton"),
                        ("var", "country")),
                    ("reducer",
                        ("var", "mean"),
                        ("cast",
                            ("var", "age"),
                            ("var", "int"))),
                    ("reducer",
                        ("var", "sum"),
                        ("cast",
                            ("var", "age"),
                            ("var", "int")))),
                params=[testlib.get_fixture_path("fake_users.csv")]),
            allow_io=True)

        # Round the output means for comparison.
        actual = []
        for row in result:
            row[1] = int(row[1])
            actual.append(row)

        expected = repeated.meld(['El Salvador', 55, 1287],
                                 ['Ethiopia', 55, 1210],
                                 ['French Guiana', 47, 381],
                                 ['Germany', 42, 299],
                                 ['Haiti', 46, 610],
                                 ['Mayotte', 50, 865],
                                 ['Portugal', 48, 485])

        self.assertItemsEqual(expected, actual)
Example #17
0
 def fixture_len(self):
     with open(testlib.get_fixture_path(self.fixture_name), "r") as fd:
         return counted.count(repeated.lines(fd))
Example #18
0
 def testIOMustBeRequested(self):
     """Test that IO isn't available unless requested."""
     with self.assertRaises(errors.EfilterError):
         api.apply("SELECT * FROM csv(?)",
                   replacements=[testlib.get_fixture_path("small.csv")])
Example #19
0
 def run(self):
     with open(testlib.get_fixture_path("hygdata_v3.csv"), "r") as fd:
         for line in csv_reader.LazyCSVReader(fd, output_dicts=True):
             if float(line["dist"]) < 300 and len(line["proper"]) > 1:
                 _ = dict(name=line["proper"], id=line["id"])
Example #20
0
 def testReading(self):
     """Test that reader reads the same lines as the File object."""
     with open(testlib.get_fixture_path("small.csv"), "r") as fd:
         reader = csv_reader.LazyCSVReader(fd)
         self.assertValuesEqual(reader, repeated.meld(*small_csv.EXPECTED))
Example #21
0
 def run(self):
     with open(testlib.get_fixture_path("hygdata_v3.csv"), "r") as fd:
         for line in csv_reader.LazyCSVReader(fd, output_dicts=True):
             if float(line["dist"]) < 300 and len(line["proper"]) > 1:
                 _ = dict(name=line["proper"], id=line["id"])
Example #22
0
 def fixture_len(self):
     with open(testlib.get_fixture_path(self.fixture_name), "r") as fd:
         return counted.count(repeated.lines(fd))
Example #23
0
 def testIOMustBeRequested(self):
     """Test that IO isn't available unless requested."""
     with self.assertRaises(errors.EfilterError):
         api.apply("SELECT * FROM csv(?)",
                   replacements=[testlib.get_fixture_path("small.csv")])
Example #24
0
File: io.py Project: google/dotty
 def testLines(self):
     lines = std_io.Lines()(testlib.get_fixture_path("names.txt"))
     self.assertEqual(counted.count(lines), 6)
     fd = lines.fd
     lines = None
     self.assertTrue(fd.closed)
Example #25
0
File: io.py Project: google/dotty
 def testCSV(self):
     csv = std_io.CSV()(testlib.get_fixture_path("small.csv"))
     self.assertEqual(counted.count(csv), 7)
     fd = csv.source.fd
     csv = None
     self.assertTrue(fd.closed)