def testMigrateClientHashEntryFromSimpleFile(self):
client_urn = self.SetupClient(0)
with self._Aff4Open(client_urn.Add("fs/os").Add("foo")) as fd:
hash_entry = rdf_crypto.Hash(md5=b"bar", sha256=b"baz")
fd.Set(fd.Schema.HASH, hash_entry)
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateClient(client_urn)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", ))
self.assertEqual(path_info.hash_entry.md5, b"bar")
self.assertEqual(path_info.hash_entry.sha256, b"baz")
def testMigrateAllClients(self):
client_urns = list(map(self.SetupClient, range(25)))
for client_urn in client_urns:
with self._Aff4Open(client_urn.Add("registry").Add("quux")) as fd:
fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"norf"))
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateAllClients()
for client_urn in client_urns:
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.REGISTRY,
components=("quux", ))
self.assertEqual(path_info.hash_entry.md5, b"norf")
def testMigrateClientStatEntryFromSimpleFile(self):
client_urn = self.SetupClient(0)
with self._Aff4Open(client_urn.Add("fs/os").Add("foo")) as fd:
stat_entry = rdf_client_fs.StatEntry(st_mode=1337, st_size=42)
fd.Set(fd.Schema.STAT, stat_entry)
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateClient(client_urn)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", ))
self.assertEqual(path_info.stat_entry.st_mode, 1337)
self.assertEqual(path_info.stat_entry.st_size, 42)
def testMigrateAllClientsSharded(self):
client_urns = list(map(self.SetupClient, range(31)))
for client_urn in client_urns:
with self._Aff4Open(client_urn.Add("fs/os").Add("bar")) as fd:
fd.Set(fd.Schema.HASH, rdf_crypto.Hash(sha256=b"baz"))
migrator = data_migration.ClientVfsMigrator()
for i in range(3):
migrator.MigrateAllClients(shard_number=(i + 1), shard_count=3)
for client_urn in client_urns:
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("bar", ))
self.assertEqual(path_info.hash_entry.sha256, b"baz")
def testMigrateAllClientsIsIndempotent(self):
client_urns = list(map(self.SetupClient, range(11)))
for client_urn in client_urns:
with self._Aff4Open(client_urn.Add("fs/os").Add("quux/norf")) as fd:
fd.Set(fd.Schema.STAT, rdf_client.StatEntry(st_size=42))
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateAllClients()
migrator.MigrateAllClients() # Should not fail in any way.
for client_urn in client_urns:
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("quux", "norf"))
self.assertEqual(path_info.stat_entry.st_size, 42)
def testMigrateClientStatFromTree(self):
client_urn = self.SetupClient(0)
with self._Aff4Open(client_urn.Add("fs/os").Add("foo/bar/baz")) as fd:
stat_entry = rdf_client.StatEntry(st_mtime=101)
fd.Set(fd.Schema.STAT, stat_entry)
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateClient(client_urn)
path_infos = data_store.REL_DB.ReadPathInfos(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components_list=[("foo",), ("foo", "bar"), ("foo", "bar", "baz")])
self.assertEqual(path_infos[("foo",)].stat_entry.st_mtime, None)
self.assertEqual(path_infos[("foo", "bar")].stat_entry.st_mtime, None)
self.assertEqual(path_infos[("foo", "bar", "baz")].stat_entry.st_mtime, 101)
def testMigrateClientStatAndHashEntryFromSimpleFile(self):
client_urn = self.SetupClient(0)
with self._Aff4Open(client_urn.Add("fs/os").Add("foo")) as fd:
stat_entry = rdf_client_fs.StatEntry(st_mode=108)
fd.Set(fd.Schema.STAT, stat_entry)
hash_entry = rdf_crypto.Hash(sha256=b"quux")
fd.Set(fd.Schema.HASH, hash_entry)
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateClient(client_urn)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", ))
self.assertEqual(path_info.stat_entry.st_mode, 108)
self.assertEqual(path_info.hash_entry.sha256, b"quux")
def testMigrateClientsSmallThreadCount(self):
client_urns = list(map(self.SetupClient, range(25)))
for i, client_urn in enumerate(client_urns):
with self._Aff4Open(client_urn.Add("fs/os").Add("foo").Add(str(i))) as fd:
fd.Set(fd.Schema.STAT, rdf_client.StatEntry(st_size=i + 42))
fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"bar"))
migrator = data_migration.ClientVfsMigrator()
migrator.thread_count = 3
migrator.MigrateClients(client_urns)
for i, client_urn in enumerate(client_urns):
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", unicode(i)))
self.assertEqual(path_info.hash_entry.md5, b"bar")
self.assertEqual(path_info.stat_entry.st_size, i + 42)
def testMigrateClientHashHistory(self):
datetime = rdfvalue.RDFDatetime.FromHumanReadable
client_urn = self.SetupClient(0)
file_urn = client_urn.Add("fs/os").Add("bar")
with test_lib.FakeTime(datetime("2010-01-01")):
with self._Aff4Open(file_urn) as fd:
fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"quux"))
with test_lib.FakeTime(datetime("2020-01-01")):
with self._Aff4Open(file_urn) as fd:
fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"norf"))
with test_lib.FakeTime(datetime("2030-01-01")):
with self._Aff4Open(file_urn) as fd:
fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"blargh"))
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateClient(client_urn)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("bar", ),
timestamp=datetime("2010-12-31"))
self.assertEqual(path_info.hash_entry.md5, b"quux")
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("bar", ),
timestamp=datetime("2020-12-31"))
self.assertEqual(path_info.hash_entry.md5, b"norf")
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("bar", ),
timestamp=datetime("2030-12-31"))
self.assertEqual(path_info.hash_entry.md5, b"blargh")
def testMigrateClientStatHistory(self):
datetime = rdfvalue.RDFDatetime.FromHumanReadable
client_urn = self.SetupClient(0)
file_urn = client_urn.Add("fs/os").Add("foo")
with test_lib.FakeTime(datetime("2000-01-01")):
with self._Aff4Open(file_urn) as fd:
fd.Set(fd.Schema.STAT, rdf_client_fs.StatEntry(st_size=10))
with test_lib.FakeTime(datetime("2000-02-02")):
with self._Aff4Open(file_urn) as fd:
fd.Set(fd.Schema.STAT, rdf_client_fs.StatEntry(st_size=20))
with test_lib.FakeTime(datetime("2000-03-03")):
with self._Aff4Open(file_urn) as fd:
fd.Set(fd.Schema.STAT, rdf_client_fs.StatEntry(st_size=30))
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateClient(client_urn)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", ),
timestamp=datetime("2000-01-10"))
self.assertEqual(path_info.stat_entry.st_size, 10)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", ),
timestamp=datetime("2000-02-20"))
self.assertEqual(path_info.stat_entry.st_size, 20)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", ),
timestamp=datetime("2000-03-30"))
self.assertEqual(path_info.stat_entry.st_size, 30)
def testMigrateClientsSmallClientBatchSize(self):
client_urns = list(map(self.SetupClient, range(25)))
for client_urn in client_urns:
with self._Aff4Open(client_urn.Add("fs/tsk").Add("bar/baz")) as fd:
fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"quux"))
migrator = data_migration.ClientVfsMigrator()
migrator.client_batch_size = 8
migrator.MigrateClients(client_urns)
for client_urn in client_urns:
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.TSK,
components=("bar", ))
self.assertIsNone(path_info.hash_entry.md5)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.TSK,
components=("bar", "baz"))
self.assertEqual(path_info.hash_entry.md5, b"quux")
def testMigrateClientWithSmallVfsGroupSize(self):
client_urn = self.SetupClient(0)
file_urn = client_urn.Add("fs/os").Add("/".join(["foo"] * 42))
with self._Aff4Open(file_urn) as fd:
fd.Set(fd.Schema.HASH, rdf_crypto.Hash(md5=b"quux"))
migrator = data_migration.ClientVfsMigrator()
migrator.vfs_group_size = 5
migrator.MigrateClient(client_urn)
for i in range(42):
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", ) * i)
self.assertIsNone(path_info.hash_entry.md5)
path_info = data_store.REL_DB.ReadPathInfo(
client_id=client_urn.Basename(),
path_type=rdf_objects.PathInfo.PathType.OS,
components=("foo", ) * 42)
self.assertEqual(path_info.hash_entry.md5, b"quux")
def Start(self):
super(ClientVfsMigrationFlow, self).Start()
migrator = data_migration.ClientVfsMigrator()
migrator.MigrateClient(client_urn=self.client_urn)
