def testMigrateClientHashEntryFromSimpleFile(self): client_urn = self.SetupClient(0) with self._Aff4Open(client_urn.Add("fs/os").Add("foo")) as fd: hash_entry = rdf_crypto.Hash(md5=b"bar", sha256=b"baz") fd.Set(fd.Schema.HASH, hash_entry) migrator = data_migration.ClientVfsMigrator() migrator.MigrateClient(client_urn) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", )) self.assertEqual(path_info.hash_entry.md5, b"bar") self.assertEqual(path_info.hash_entry.sha256, b"baz")
def testMigrateAllClients(self): client_urns = list(map(self.SetupClient, range(25))) for client_urn in client_urns: with self._Aff4Open(client_urn.Add("registry").Add("quux")) as fd: fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"norf")) migrator = data_migration.ClientVfsMigrator() migrator.MigrateAllClients() for client_urn in client_urns: path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.REGISTRY, components=("quux", )) self.assertEqual(path_info.hash_entry.md5, b"norf")
def testMigrateClientStatEntryFromSimpleFile(self): client_urn = self.SetupClient(0) with self._Aff4Open(client_urn.Add("fs/os").Add("foo")) as fd: stat_entry = rdf_client_fs.StatEntry(st_mode=1337, st_size=42) fd.Set(fd.Schema.STAT, stat_entry) migrator = data_migration.ClientVfsMigrator() migrator.MigrateClient(client_urn) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", )) self.assertEqual(path_info.stat_entry.st_mode, 1337) self.assertEqual(path_info.stat_entry.st_size, 42)
def testMigrateAllClientsSharded(self): client_urns = list(map(self.SetupClient, range(31))) for client_urn in client_urns: with self._Aff4Open(client_urn.Add("fs/os").Add("bar")) as fd: fd.Set(fd.Schema.HASH, rdf_crypto.Hash(sha256=b"baz")) migrator = data_migration.ClientVfsMigrator() for i in range(3): migrator.MigrateAllClients(shard_number=(i + 1), shard_count=3) for client_urn in client_urns: path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("bar", )) self.assertEqual(path_info.hash_entry.sha256, b"baz")
def testMigrateAllClientsIsIndempotent(self): client_urns = list(map(self.SetupClient, range(11))) for client_urn in client_urns: with self._Aff4Open(client_urn.Add("fs/os").Add("quux/norf")) as fd: fd.Set(fd.Schema.STAT, rdf_client.StatEntry(st_size=42)) migrator = data_migration.ClientVfsMigrator() migrator.MigrateAllClients() migrator.MigrateAllClients() # Should not fail in any way. for client_urn in client_urns: path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("quux", "norf")) self.assertEqual(path_info.stat_entry.st_size, 42)
def testMigrateClientStatFromTree(self): client_urn = self.SetupClient(0) with self._Aff4Open(client_urn.Add("fs/os").Add("foo/bar/baz")) as fd: stat_entry = rdf_client.StatEntry(st_mtime=101) fd.Set(fd.Schema.STAT, stat_entry) migrator = data_migration.ClientVfsMigrator() migrator.MigrateClient(client_urn) path_infos = data_store.REL_DB.ReadPathInfos( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components_list=[("foo",), ("foo", "bar"), ("foo", "bar", "baz")]) self.assertEqual(path_infos[("foo",)].stat_entry.st_mtime, None) self.assertEqual(path_infos[("foo", "bar")].stat_entry.st_mtime, None) self.assertEqual(path_infos[("foo", "bar", "baz")].stat_entry.st_mtime, 101)
def testMigrateClientStatAndHashEntryFromSimpleFile(self): client_urn = self.SetupClient(0) with self._Aff4Open(client_urn.Add("fs/os").Add("foo")) as fd: stat_entry = rdf_client_fs.StatEntry(st_mode=108) fd.Set(fd.Schema.STAT, stat_entry) hash_entry = rdf_crypto.Hash(sha256=b"quux") fd.Set(fd.Schema.HASH, hash_entry) migrator = data_migration.ClientVfsMigrator() migrator.MigrateClient(client_urn) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", )) self.assertEqual(path_info.stat_entry.st_mode, 108) self.assertEqual(path_info.hash_entry.sha256, b"quux")
def testMigrateClientsSmallThreadCount(self): client_urns = list(map(self.SetupClient, range(25))) for i, client_urn in enumerate(client_urns): with self._Aff4Open(client_urn.Add("fs/os").Add("foo").Add(str(i))) as fd: fd.Set(fd.Schema.STAT, rdf_client.StatEntry(st_size=i + 42)) fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"bar")) migrator = data_migration.ClientVfsMigrator() migrator.thread_count = 3 migrator.MigrateClients(client_urns) for i, client_urn in enumerate(client_urns): path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", unicode(i))) self.assertEqual(path_info.hash_entry.md5, b"bar") self.assertEqual(path_info.stat_entry.st_size, i + 42)
def testMigrateClientHashHistory(self): datetime = rdfvalue.RDFDatetime.FromHumanReadable client_urn = self.SetupClient(0) file_urn = client_urn.Add("fs/os").Add("bar") with test_lib.FakeTime(datetime("2010-01-01")): with self._Aff4Open(file_urn) as fd: fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"quux")) with test_lib.FakeTime(datetime("2020-01-01")): with self._Aff4Open(file_urn) as fd: fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"norf")) with test_lib.FakeTime(datetime("2030-01-01")): with self._Aff4Open(file_urn) as fd: fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"blargh")) migrator = data_migration.ClientVfsMigrator() migrator.MigrateClient(client_urn) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("bar", ), timestamp=datetime("2010-12-31")) self.assertEqual(path_info.hash_entry.md5, b"quux") path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("bar", ), timestamp=datetime("2020-12-31")) self.assertEqual(path_info.hash_entry.md5, b"norf") path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("bar", ), timestamp=datetime("2030-12-31")) self.assertEqual(path_info.hash_entry.md5, b"blargh")
def testMigrateClientStatHistory(self): datetime = rdfvalue.RDFDatetime.FromHumanReadable client_urn = self.SetupClient(0) file_urn = client_urn.Add("fs/os").Add("foo") with test_lib.FakeTime(datetime("2000-01-01")): with self._Aff4Open(file_urn) as fd: fd.Set(fd.Schema.STAT, rdf_client_fs.StatEntry(st_size=10)) with test_lib.FakeTime(datetime("2000-02-02")): with self._Aff4Open(file_urn) as fd: fd.Set(fd.Schema.STAT, rdf_client_fs.StatEntry(st_size=20)) with test_lib.FakeTime(datetime("2000-03-03")): with self._Aff4Open(file_urn) as fd: fd.Set(fd.Schema.STAT, rdf_client_fs.StatEntry(st_size=30)) migrator = data_migration.ClientVfsMigrator() migrator.MigrateClient(client_urn) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", ), timestamp=datetime("2000-01-10")) self.assertEqual(path_info.stat_entry.st_size, 10) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", ), timestamp=datetime("2000-02-20")) self.assertEqual(path_info.stat_entry.st_size, 20) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", ), timestamp=datetime("2000-03-30")) self.assertEqual(path_info.stat_entry.st_size, 30)
def testMigrateClientsSmallClientBatchSize(self): client_urns = list(map(self.SetupClient, range(25))) for client_urn in client_urns: with self._Aff4Open(client_urn.Add("fs/tsk").Add("bar/baz")) as fd: fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"quux")) migrator = data_migration.ClientVfsMigrator() migrator.client_batch_size = 8 migrator.MigrateClients(client_urns) for client_urn in client_urns: path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.TSK, components=("bar", )) self.assertIsNone(path_info.hash_entry.md5) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.TSK, components=("bar", "baz")) self.assertEqual(path_info.hash_entry.md5, b"quux")
def testMigrateClientWithSmallVfsGroupSize(self): client_urn = self.SetupClient(0) file_urn = client_urn.Add("fs/os").Add("/".join(["foo"] * 42)) with self._Aff4Open(file_urn) as fd: fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"quux")) migrator = data_migration.ClientVfsMigrator() migrator.vfs_group_size = 5 migrator.MigrateClient(client_urn) for i in range(42): path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", ) * i) self.assertIsNone(path_info.hash_entry.md5) path_info = data_store.REL_DB.ReadPathInfo( client_id=client_urn.Basename(), path_type=rdf_objects.PathInfo.PathType.OS, components=("foo", ) * 42) self.assertEqual(path_info.hash_entry.md5, b"quux")
def Start(self): super(ClientVfsMigrationFlow, self).Start() migrator = data_migration.ClientVfsMigrator() migrator.MigrateClient(client_urn=self.client_urn)