def run_test( source, expected_data ):
destination = ClientMigration.MigrationDestinationListPairs( self )
job = ClientMigration.MigrationJob( self, 'test', source, destination )
job.Run()
self.assertEqual( set( destination.GetDataReceived() ), set( expected_data ) )
def _test_mappings_list_to_list(self):
data = list(self._hashes_to_current_tags.items())
self.assertTrue(len(data) > 0)
source = ClientMigration.MigrationSourceList(self, data)
destination = ClientMigration.MigrationDestinationListMappings(self)
job = ClientMigration.MigrationJob(self, 'test', source, destination)
job.Run()
self.assertEqual(destination.GetDataReceived(), data)
def run_test( source, destination_path, content_type, expected_data ):
destination = ClientMigration.MigrationDestinationHTPA( self, destination_path, content_type )
job = ClientMigration.MigrationJob( self, 'test', source, destination )
job.Run()
hta = HydrusTagArchive.HydrusTagPairArchive( destination_path )
result = list( hta.IteratePairs() )
self.assertEqual( set( result ), set( expected_data ) )
hta.Close()
def run_test( source, destination_path, desired_hash_type, expected_data ):
destination = ClientMigration.MigrationDestinationHTA( self, destination_path, desired_hash_type )
job = ClientMigration.MigrationJob( self, 'test', source, destination )
job.Run()
hta = HydrusTagArchive.HydrusTagArchive( destination_path )
result = list( hta.IterateMappings() )
self.assertEqual( dict( result ), dict( expected_data ) )
hta.Close()
def run_test(source, tag_service_key, content_action, expected_data):
destination = ClientMigration.MigrationDestinationTagServicePairs(
self, tag_service_key, content_action, content_type)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
if content_type == HC.CONTENT_TYPE_TAG_PARENTS:
statuses_to_pairs = self.Read('tag_parents', tag_service_key)
elif content_type == HC.CONTENT_TYPE_TAG_SIBLINGS:
statuses_to_pairs = self.Read('tag_siblings', tag_service_key)
if content_action == HC.CONTENT_UPDATE_ADD:
should_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_CURRENT])
should_not_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_DELETED])
elif content_action == HC.CONTENT_UPDATE_DELETE:
should_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_DELETED])
should_not_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_CURRENT])
elif content_action == HC.CONTENT_UPDATE_PEND:
should_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_PENDING])
should_not_be_in = set()
elif content_action == HC.CONTENT_UPDATE_PETITION:
should_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_PETITIONED])
should_not_be_in = set()
for pair in expected_data:
self.assertIn(pair, should_be_in)
self.assertNotIn(pair, should_not_be_in)
def _test_pairs_list_to_htpa(self, content_type):
def run_test(source, destination_path, content_type, expected_data):
destination = ClientMigration.MigrationDestinationHTPA(
self, destination_path, content_type)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
hta = HydrusTagArchive.HydrusTagPairArchive(destination_path)
result = list(hta.IteratePairs())
self.assertEqual(set(result), set(expected_data))
hta.Close()
(current, pending, to_be_pended,
deleted) = pair_types_to_pools[content_type]
htpa_path = os.path.join(TestController.DB_DIR, 'htpa.db')
#
source = ClientMigration.MigrationSourceList(self, current)
run_test(source, htpa_path, content_type, list(current))
#
os.remove(htpa_path)
def _test_pairs_list_to_list( self, content_type ):
( current, pending, to_be_pended, deleted ) = pair_types_to_pools[ content_type ]
data = list( current )
self.assertTrue( len( data ) > 0 )
source = ClientMigration.MigrationSourceList( self, data )
destination = ClientMigration.MigrationDestinationListPairs( self )
job = ClientMigration.MigrationJob( self, 'test', source, destination )
job.Run()
self.assertEqual( destination.GetDataReceived(), data )
def _test_pairs_service_to_list( self, content_type ):
def run_test( source, expected_data ):
destination = ClientMigration.MigrationDestinationListPairs( self )
job = ClientMigration.MigrationJob( self, 'test', source, destination )
job.Run()
self.assertEqual( set( destination.GetDataReceived() ), set( expected_data ) )
( current, pending, to_be_pended, deleted ) = pair_types_to_pools[ content_type ]
# test filters and content statuses
tag_repo_service_key = self._test_tag_repo_service_keys[10]
content_source_tests = []
content_source_tests.append( ( CC.DEFAULT_LOCAL_TAG_SERVICE_KEY, ( current, ), ( HC.CONTENT_STATUS_CURRENT, ) ) )
content_source_tests.append( ( CC.DEFAULT_LOCAL_TAG_SERVICE_KEY, ( deleted, ), ( HC.CONTENT_STATUS_DELETED, ) ) )
content_source_tests.append( ( tag_repo_service_key, ( current, ), ( HC.CONTENT_STATUS_CURRENT, ) ) )
content_source_tests.append( ( tag_repo_service_key, ( current, pending ), ( HC.CONTENT_STATUS_CURRENT, HC.CONTENT_STATUS_PENDING ) ) )
content_source_tests.append( ( tag_repo_service_key, ( deleted, ), ( HC.CONTENT_STATUS_DELETED, ) ) )
free_filter = ClientTags.TagFilter()
namespace_filter = ClientTags.TagFilter()
namespace_filter.SetRule( ':', CC.FILTER_WHITELIST )
namespace_filter.SetRule( '', CC.FILTER_BLACKLIST )
test_filters = []
test_filters.append( ( free_filter, free_filter ) )
test_filters.append( ( namespace_filter, free_filter ) )
test_filters.append( ( free_filter, namespace_filter ) )
test_filters.append( ( namespace_filter, namespace_filter ) )
for ( left_tag_filter, right_tag_filter ) in test_filters:
for ( service_key, content_lists, content_statuses ) in content_source_tests:
source = ClientMigration.MigrationSourceTagServicePairs( self, service_key, content_type, left_tag_filter, right_tag_filter, content_statuses )
expected_data = set()
for content_list in content_lists:
expected_data.update( ( ( left_tag, right_tag ) for ( left_tag, right_tag ) in content_list if left_tag_filter.TagOK( left_tag ) and right_tag_filter.TagOK( right_tag ) ) )
run_test( source, expected_data )
def _test_mappings_list_to_hta(self):
def run_test(source, destination_path, desired_hash_type,
expected_data):
destination = ClientMigration.MigrationDestinationHTA(
self, destination_path, desired_hash_type)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
hta = HydrusTagArchive.HydrusTagArchive(destination_path)
result = list(hta.IterateMappings())
self.assertEqual(dict(result), dict(expected_data))
hta.Close()
md5_hta_path = os.path.join(TestController.DB_DIR, 'md5hta.db')
sha256_hta_path = os.path.join(TestController.DB_DIR, 'sha256hta.db')
#
md5_data = [(self._sha256_to_md5[hash], tags)
for (hash, tags) in self._hashes_to_current_tags.items()]
sha256_data = list(self._hashes_to_current_tags.items())
md5_source = ClientMigration.MigrationSourceList(self, md5_data)
sha256_source = ClientMigration.MigrationSourceList(self, sha256_data)
run_test(md5_source, md5_hta_path, 'md5', md5_data)
run_test(sha256_source, sha256_hta_path, 'sha256', sha256_data)
#
os.remove(md5_hta_path)
os.remove(sha256_hta_path)
def _test_pairs_htpa_to_list(self, content_type):
def run_test(source, expected_data):
destination = ClientMigration.MigrationDestinationListPairs(self)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
self.assertEqual(set(destination.GetDataReceived()),
set(expected_data))
(current, pending, to_be_pended,
deleted) = pair_types_to_pools[content_type]
htpa_path = os.path.join(TestController.DB_DIR, 'htpa.db')
htpa = HydrusTagArchive.HydrusTagPairArchive(htpa_path)
if content_type == HC.CONTENT_TYPE_TAG_PARENTS:
htpa.SetPairType(HydrusTagArchive.TAG_PAIR_TYPE_PARENTS)
elif content_type == HC.CONTENT_TYPE_TAG_SIBLINGS:
htpa.SetPairType(HydrusTagArchive.TAG_PAIR_TYPE_SIBLINGS)
htpa.BeginBigJob()
htpa.AddPairs(current)
htpa.CommitBigJob()
htpa.Optimise()
htpa.Close()
del htpa
#
# test tag filter, left, right, both
free_filter = HydrusTags.TagFilter()
namespace_filter = HydrusTags.TagFilter()
namespace_filter.SetRule(':', HC.FILTER_WHITELIST)
namespace_filter.SetRule('', HC.FILTER_BLACKLIST)
test_filters = []
test_filters.append((free_filter, free_filter))
test_filters.append((namespace_filter, free_filter))
test_filters.append((free_filter, namespace_filter))
test_filters.append((namespace_filter, namespace_filter))
for (left_tag_filter, right_tag_filter) in test_filters:
source = ClientMigration.MigrationSourceHTPA(
self, htpa_path, left_tag_filter, right_tag_filter)
expected_data = [(left_tag, right_tag)
for (left_tag, right_tag) in current
if left_tag_filter.TagOK(left_tag)
and right_tag_filter.TagOK(right_tag)]
run_test(source, expected_data)
#
os.remove(htpa_path)
def run_test(source, tag_service_key, content_action, expected_data):
destination = ClientMigration.MigrationDestinationTagServiceMappings(
self, tag_service_key, content_action)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
self._db._weakref_media_result_cache = ClientMediaResultCache.MediaResultCache(
)
hashes_to_media_results = {
media_result.GetHash(): media_result
for media_result in self.Read(
'media_results', list(self._hashes_to_current_tags.keys()))
}
for (hash, tags) in expected_data:
media_result = hashes_to_media_results[hash]
t_m = media_result.GetTagsManager()
if content_action == HC.CONTENT_UPDATE_ADD:
current_tags = t_m.GetCurrent(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertIn(tag, current_tags)
elif content_action == HC.CONTENT_UPDATE_DELETE:
current_tags = t_m.GetCurrent(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
deleted_tags = t_m.GetDeleted(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertNotIn(tag, current_tags)
self.assertIn(tag, deleted_tags)
elif content_action == HC.CONTENT_UPDATE_CLEAR_DELETE_RECORD:
deleted_tags = t_m.GetDeleted(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertNotIn(tag, deleted_tags)
elif content_action == HC.CONTENT_UPDATE_PEND:
pending_tags = t_m.GetPending(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertIn(tag, pending_tags)
elif content_action == HC.CONTENT_UPDATE_PETITION:
petitioned_tags = t_m.GetPetitioned(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertIn(tag, petitioned_tags)
def _test_mappings_list_to_service(self):
def run_test(source, tag_service_key, content_action, expected_data):
destination = ClientMigration.MigrationDestinationTagServiceMappings(
self, tag_service_key, content_action)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
self._db._weakref_media_result_cache = ClientMediaResultCache.MediaResultCache(
)
hashes_to_media_results = {
media_result.GetHash(): media_result
for media_result in self.Read(
'media_results', list(self._hashes_to_current_tags.keys()))
}
for (hash, tags) in expected_data:
media_result = hashes_to_media_results[hash]
t_m = media_result.GetTagsManager()
if content_action == HC.CONTENT_UPDATE_ADD:
current_tags = t_m.GetCurrent(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertIn(tag, current_tags)
elif content_action == HC.CONTENT_UPDATE_DELETE:
current_tags = t_m.GetCurrent(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
deleted_tags = t_m.GetDeleted(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertNotIn(tag, current_tags)
self.assertIn(tag, deleted_tags)
elif content_action == HC.CONTENT_UPDATE_CLEAR_DELETE_RECORD:
deleted_tags = t_m.GetDeleted(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertNotIn(tag, deleted_tags)
elif content_action == HC.CONTENT_UPDATE_PEND:
pending_tags = t_m.GetPending(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertIn(tag, pending_tags)
elif content_action == HC.CONTENT_UPDATE_PETITION:
petitioned_tags = t_m.GetPetitioned(
tag_service_key, ClientTags.TAG_DISPLAY_STORAGE)
for tag in tags:
self.assertIn(tag, petitioned_tags)
#
# local add
data = [(hash, set(random.sample(to_be_pended_tag_pool, 2)))
for hash in self._hashes_to_current_tags.keys()]
source = ClientMigration.MigrationSourceList(self, data)
run_test(source, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
HC.CONTENT_UPDATE_ADD, data)
# local delete
data = [(hash, set(random.sample(tags, 2)))
for (hash, tags) in self._hashes_to_current_tags.items()]
source = ClientMigration.MigrationSourceList(self, data)
run_test(source, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
HC.CONTENT_UPDATE_DELETE, data)
# local clear deletion record
data = [(hash, set(random.sample(tags, 2)))
for (hash, tags) in self._hashes_to_deleted_tags.items()]
source = ClientMigration.MigrationSourceList(self, data)
run_test(source, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
HC.CONTENT_UPDATE_CLEAR_DELETE_RECORD, data)
# tag repo pend
data = [(hash, set(random.sample(to_be_pended_tag_pool, 2)))
for hash in self._hashes_to_current_tags.keys()]
source = ClientMigration.MigrationSourceList(self, data)
run_test(source, self._test_tag_repo_service_keys[1],
HC.CONTENT_UPDATE_PEND, data)
# tag repo petition
data = [(hash, set(random.sample(tags, 2)))
for (hash, tags) in self._hashes_to_current_tags.items()]
source = ClientMigration.MigrationSourceList(self, data)
run_test(source, self._test_tag_repo_service_keys[1],
HC.CONTENT_UPDATE_PETITION, data)
def _test_mappings_service_to_list(self):
def run_test(source, expected_data):
destination = ClientMigration.MigrationDestinationListMappings(
self)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
self.assertEqual(dict(destination.GetDataReceived()),
dict(expected_data))
# test file filter
tag_repo_service_key = self._test_tag_repo_service_keys[0]
tag_filter = HydrusTags.TagFilter()
source = ClientMigration.MigrationSourceTagServiceMappings(
self, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
CC.COMBINED_FILE_SERVICE_KEY, 'sha256', None, tag_filter,
(HC.CONTENT_STATUS_CURRENT, ))
expected_data = list(self._hashes_to_current_tags.items())
run_test(source, expected_data)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, tag_repo_service_key, CC.COMBINED_FILE_SERVICE_KEY, 'sha256',
None, tag_filter, (HC.CONTENT_STATUS_CURRENT, ))
expected_data = list(self._hashes_to_current_tags.items())
run_test(source, expected_data)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY, CC.LOCAL_FILE_SERVICE_KEY,
'sha256', None, tag_filter, (HC.CONTENT_STATUS_CURRENT, ))
expected_data = [(hash, tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in self._my_files_sha256]
run_test(source, expected_data)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, tag_repo_service_key, CC.LOCAL_FILE_SERVICE_KEY, 'sha256',
None, tag_filter, (HC.CONTENT_STATUS_CURRENT, ))
expected_data = [(hash, tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in self._my_files_sha256]
run_test(source, expected_data)
# not all hashes, since hash type lookup only available for imported files
hashes = random.sample(self._my_files_sha256, 25)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
CC.COMBINED_FILE_SERVICE_KEY, 'sha256', hashes, tag_filter,
(HC.CONTENT_STATUS_CURRENT, ))
expected_data = [(hash, tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in hashes]
run_test(source, expected_data)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, tag_repo_service_key, CC.COMBINED_FILE_SERVICE_KEY, 'sha256',
hashes, tag_filter, (HC.CONTENT_STATUS_CURRENT, ))
expected_data = [(hash, tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in hashes]
run_test(source, expected_data)
# test desired hash type
# not all hashes, since hash type lookup only available for imported files
expected_data = [(self._sha256_to_sha1[hash], tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in self._my_files_sha256]
source = ClientMigration.MigrationSourceTagServiceMappings(
self, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
CC.COMBINED_FILE_SERVICE_KEY, 'sha1', None, tag_filter,
(HC.CONTENT_STATUS_CURRENT, ))
run_test(source, expected_data)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, tag_repo_service_key, CC.COMBINED_FILE_SERVICE_KEY, 'sha1',
None, tag_filter, (HC.CONTENT_STATUS_CURRENT, ))
run_test(source, expected_data)
# tag filter
tag_filter = HydrusTags.TagFilter()
tag_filter.SetRule('', HC.FILTER_WHITELIST)
tag_filter.SetRule(':', HC.FILTER_BLACKLIST)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
CC.COMBINED_FILE_SERVICE_KEY, 'sha256', None, tag_filter,
(HC.CONTENT_STATUS_CURRENT, ))
expected_data = [(hash, tag_filter.Filter(tags))
for (hash,
tags) in self._hashes_to_current_tags.items()]
expected_data = [(hash, tags) for (hash, tags) in expected_data
if len(tags) > 0]
run_test(source, expected_data)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, tag_repo_service_key, CC.COMBINED_FILE_SERVICE_KEY, 'sha256',
None, tag_filter, (HC.CONTENT_STATUS_CURRENT, ))
expected_data = [(hash, tag_filter.Filter(tags))
for (hash,
tags) in self._hashes_to_current_tags.items()]
expected_data = [(hash, tags) for (hash, tags) in expected_data
if len(tags) > 0]
run_test(source, expected_data)
# test statuses
tag_filter = HydrusTags.TagFilter()
source = ClientMigration.MigrationSourceTagServiceMappings(
self, CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
CC.COMBINED_FILE_SERVICE_KEY, 'sha256', None, tag_filter,
(HC.CONTENT_STATUS_DELETED, ))
expected_data = list(self._hashes_to_deleted_tags.items())
run_test(source, expected_data)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, tag_repo_service_key, CC.COMBINED_FILE_SERVICE_KEY, 'sha256',
None, tag_filter, (HC.CONTENT_STATUS_DELETED, ))
expected_data = list(self._hashes_to_deleted_tags.items())
run_test(source, expected_data)
source = ClientMigration.MigrationSourceTagServiceMappings(
self, tag_repo_service_key, CC.COMBINED_FILE_SERVICE_KEY, 'sha256',
None, tag_filter,
(HC.CONTENT_STATUS_CURRENT, HC.CONTENT_STATUS_PENDING))
expected_data = collections.defaultdict(set)
for (hash, tags) in self._hashes_to_current_tags.items():
expected_data[hash].update(tags)
for (hash, tags) in self._hashes_to_pending_tags.items():
expected_data[hash].update(tags)
expected_data = list(expected_data.items())
run_test(source, expected_data)
def _test_mappings_hta_to_list(self):
def run_test(source, expected_data):
destination = ClientMigration.MigrationDestinationListMappings(
self)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
self.assertEqual(dict(destination.GetDataReceived()),
dict(expected_data))
md5_hta_path = os.path.join(TestController.DB_DIR, 'md5hta.db')
sha256_hta_path = os.path.join(TestController.DB_DIR, 'sha256hta.db')
md5_hta = HydrusTagArchive.HydrusTagArchive(md5_hta_path)
sha256_hta = HydrusTagArchive.HydrusTagArchive(sha256_hta_path)
md5_hta.SetHashType(HydrusTagArchive.HASH_TYPE_MD5)
sha256_hta.SetHashType(HydrusTagArchive.HASH_TYPE_SHA256)
md5_hta.BeginBigJob()
sha256_hta.BeginBigJob()
for (hash, tags) in self._hashes_to_current_tags.items():
md5 = self._sha256_to_md5[hash]
md5_hta.AddMappings(md5, tags)
sha256_hta.AddMappings(hash, tags)
md5_hta.CommitBigJob()
sha256_hta.CommitBigJob()
md5_hta.Optimise()
sha256_hta.Optimise()
md5_hta.Close()
sha256_hta.Close()
del md5_hta
del sha256_hta
#
# test file filter
tag_filter = HydrusTags.TagFilter()
source = ClientMigration.MigrationSourceHTA(
self, md5_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'md5', None,
tag_filter)
expected_data = [(self._sha256_to_md5[hash], tags)
for (hash,
tags) in self._hashes_to_current_tags.items()]
run_test(source, expected_data)
source = ClientMigration.MigrationSourceHTA(
self, sha256_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'sha256',
None, tag_filter)
expected_data = list(self._hashes_to_current_tags.items())
run_test(source, expected_data)
source = ClientMigration.MigrationSourceHTA(self, md5_hta_path,
CC.LOCAL_FILE_SERVICE_KEY,
'md5', None, tag_filter)
expected_data = [(self._sha256_to_md5[hash], tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in self._my_files_sha256]
run_test(source, expected_data)
source = ClientMigration.MigrationSourceHTA(self, sha256_hta_path,
CC.LOCAL_FILE_SERVICE_KEY,
'sha256', None, tag_filter)
expected_data = [(hash, tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in self._my_files_sha256]
run_test(source, expected_data)
# not all hashes, since hash type lookup only available for imported files
hashes = random.sample(self._my_files_sha256, 25)
source = ClientMigration.MigrationSourceHTA(
self, md5_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'md5', hashes,
tag_filter)
expected_data = [(self._sha256_to_md5[hash], tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in hashes]
run_test(source, expected_data)
source = ClientMigration.MigrationSourceHTA(
self, sha256_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'sha256',
hashes, tag_filter)
expected_data = [(hash, tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in hashes]
run_test(source, expected_data)
# test desired hash type
# not all hashes, since hash type lookup only available for imported files
expected_data = [(self._sha256_to_sha1[hash], tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in self._my_files_sha256]
source = ClientMigration.MigrationSourceHTA(
self, md5_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'sha1', None,
tag_filter)
run_test(source, expected_data)
source = ClientMigration.MigrationSourceHTA(
self, sha256_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'sha1', None,
tag_filter)
run_test(source, expected_data)
# do a test with specific hashes, so md5->sha1 does interim sha256 conversion
# not all hashes, since hash type lookup only available for imported files
hashes = random.sample(self._my_files_sha256, 25)
expected_data = [(self._sha256_to_sha1[hash], tags)
for (hash,
tags) in self._hashes_to_current_tags.items()
if hash in hashes]
source = ClientMigration.MigrationSourceHTA(
self, md5_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'sha1', hashes,
tag_filter)
run_test(source, expected_data)
# tag filter
tag_filter = HydrusTags.TagFilter()
tag_filter.SetRule('', HC.FILTER_WHITELIST)
tag_filter.SetRule(':', HC.FILTER_BLACKLIST)
source = ClientMigration.MigrationSourceHTA(
self, md5_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'md5', None,
tag_filter)
expected_data = [(self._sha256_to_md5[hash], tag_filter.Filter(tags))
for (hash,
tags) in self._hashes_to_current_tags.items()]
expected_data = [(hash, tags) for (hash, tags) in expected_data
if len(tags) > 0]
run_test(source, expected_data)
source = ClientMigration.MigrationSourceHTA(
self, sha256_hta_path, CC.COMBINED_FILE_SERVICE_KEY, 'sha256',
None, tag_filter)
expected_data = [(hash, tag_filter.Filter(tags))
for (hash,
tags) in self._hashes_to_current_tags.items()]
expected_data = [(hash, tags) for (hash, tags) in expected_data
if len(tags) > 0]
run_test(source, expected_data)
#
os.remove(md5_hta_path)
os.remove(sha256_hta_path)
def _test_pairs_list_to_service(self, content_type):
def run_test(source, tag_service_key, content_action, expected_data):
destination = ClientMigration.MigrationDestinationTagServicePairs(
self, tag_service_key, content_action, content_type)
job = ClientMigration.MigrationJob(self, 'test', source,
destination)
job.Run()
if content_type == HC.CONTENT_TYPE_TAG_PARENTS:
statuses_to_pairs = self.Read('tag_parents', tag_service_key)
elif content_type == HC.CONTENT_TYPE_TAG_SIBLINGS:
statuses_to_pairs = self.Read('tag_siblings', tag_service_key)
if content_action == HC.CONTENT_UPDATE_ADD:
should_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_CURRENT])
should_not_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_DELETED])
elif content_action == HC.CONTENT_UPDATE_DELETE:
should_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_DELETED])
should_not_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_CURRENT])
elif content_action == HC.CONTENT_UPDATE_PEND:
should_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_PENDING])
should_not_be_in = set()
elif content_action == HC.CONTENT_UPDATE_PETITION:
should_be_in = set(
statuses_to_pairs[HC.CONTENT_STATUS_PETITIONED])
should_not_be_in = set()
for pair in expected_data:
self.assertIn(pair, should_be_in)
self.assertNotIn(pair, should_not_be_in)
#
tag_repo_service_key = self._test_tag_repo_service_keys[11]
(current, pending, to_be_pended,
deleted) = pair_types_to_pools[content_type]
test_rows = []
test_rows.append((CC.DEFAULT_LOCAL_TAG_SERVICE_KEY, to_be_pended,
HC.CONTENT_UPDATE_ADD))
test_rows.append((CC.DEFAULT_LOCAL_TAG_SERVICE_KEY,
random.sample(current, 3), HC.CONTENT_UPDATE_DELETE))
test_rows.append(
(tag_repo_service_key, to_be_pended, HC.CONTENT_UPDATE_PEND))
test_rows.append((tag_repo_service_key, random.sample(current, 3),
HC.CONTENT_UPDATE_PETITION))
for (service_key, data, action) in test_rows:
source = ClientMigration.MigrationSourceList(self, data)
run_test(source, service_key, action, data)
