def RegeneratePending(self, tag_service_id, status_hook=None):
(current_mappings_table_name, deleted_mappings_table_name,
pending_mappings_table_name, petitioned_mappings_table_name
) = ClientDBMappingsStorage.GenerateMappingsTableNames(tag_service_id)
if status_hook is not None:
message = 'clearing old combined display data'
status_hook(message)
all_pending_storage_tag_ids = self._STS(
self._Execute('SELECT DISTINCT tag_id FROM {};'.format(
pending_mappings_table_name)))
self.modules_mappings_counts.ClearCounts(
ClientTags.TAG_DISPLAY_STORAGE,
self.modules_services.combined_file_service_id,
tag_service_id,
keep_current=True)
counts_cache_changes = []
num_to_do = len(all_pending_storage_tag_ids)
for (i, storage_tag_id) in enumerate(all_pending_storage_tag_ids):
if i % 100 == 0 and status_hook is not None:
message = 'regenerating pending tags {}'.format(
HydrusData.ConvertValueRangeToPrettyString(
i + 1, num_to_do))
status_hook(message)
(pending_delta, ) = self._Execute(
'SELECT COUNT( DISTINCT hash_id ) FROM {} WHERE tag_id = ?;'.
format(pending_mappings_table_name),
(storage_tag_id, )).fetchone()
counts_cache_changes.append((storage_tag_id, 0, pending_delta))
self.modules_mappings_counts_update.AddCounts(
ClientTags.TAG_DISPLAY_STORAGE,
self.modules_services.combined_file_service_id, tag_service_id,
counts_cache_changes)
self.modules_mappings_cache_combined_files_display.RegeneratePending(
tag_service_id, status_hook=status_hook)
def Generate(self, tag_service_id):
self.modules_mappings_counts.CreateTables(
ClientTags.TAG_DISPLAY_STORAGE,
self.modules_services.combined_file_service_id, tag_service_id)
#
(current_mappings_table_name, deleted_mappings_table_name,
pending_mappings_table_name, petitioned_mappings_table_name
) = ClientDBMappingsStorage.GenerateMappingsTableNames(tag_service_id)
current_mappings_exist = self._Execute(
'SELECT 1 FROM ' + current_mappings_table_name +
' LIMIT 1;').fetchone() is not None
pending_mappings_exist = self._Execute(
'SELECT 1 FROM ' + pending_mappings_table_name +
' LIMIT 1;').fetchone() is not None
if current_mappings_exist or pending_mappings_exist: # not worth iterating through all known tags for an empty service
for (group_of_ids, num_done,
num_to_do) in HydrusDB.ReadLargeIdQueryInSeparateChunks(
self._c, 'SELECT tag_id FROM tags;',
10000): # must be a cleverer way of doing this
with self._MakeTemporaryIntegerTable(
group_of_ids, 'tag_id') as temp_table_name:
current_counter = collections.Counter()
# temp tags to mappings
for (tag_id, count) in self._Execute(
'SELECT tag_id, COUNT( * ) FROM {} CROSS JOIN {} USING ( tag_id ) GROUP BY ( tag_id );'
.format(temp_table_name,
current_mappings_table_name)):
current_counter[tag_id] = count
pending_counter = collections.Counter()
# temp tags to mappings
for (tag_id, count) in self._Execute(
'SELECT tag_id, COUNT( * ) FROM {} CROSS JOIN {} USING ( tag_id ) GROUP BY ( tag_id );'
.format(temp_table_name,
pending_mappings_table_name)):
pending_counter[tag_id] = count
all_ids_seen = set(current_counter.keys())
all_ids_seen.update(pending_counter.keys())
counts_cache_changes = [(tag_id, current_counter[tag_id],
pending_counter[tag_id])
for tag_id in all_ids_seen]
if len(counts_cache_changes) > 0:
self.modules_mappings_counts_update.AddCounts(
ClientTags.TAG_DISPLAY_STORAGE,
self.modules_services.combined_file_service_id,
tag_service_id, counts_cache_changes)
self.modules_mappings_cache_combined_files_display.Generate(
tag_service_id)
def RegeneratePending( self, file_service_id, tag_service_id, status_hook = None ):
( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( tag_service_id )
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
if status_hook is not None:
message = 'clearing old specific data'
status_hook( message )
all_pending_storage_tag_ids = self._STS( self._Execute( 'SELECT DISTINCT tag_id FROM {};'.format( pending_mappings_table_name ) ) )
self.modules_mappings_counts.ClearCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, keep_current = True )
self._Execute( 'DELETE FROM {};'.format( cache_pending_mappings_table_name ) )
counts_cache_changes = []
num_to_do = len( all_pending_storage_tag_ids )
select_table_join = self.modules_files_storage.GetTableJoinLimitedByFileDomain( file_service_id, pending_mappings_table_name, HC.CONTENT_STATUS_CURRENT )
for ( i, storage_tag_id ) in enumerate( all_pending_storage_tag_ids ):
if i % 100 == 0 and status_hook is not None:
message = 'regenerating pending tags {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, num_to_do ) )
status_hook( message )
self._Execute( 'INSERT OR IGNORE INTO {} ( tag_id, hash_id ) SELECT tag_id, hash_id FROM {} WHERE tag_id = ?;'.format( cache_pending_mappings_table_name, select_table_join ), ( storage_tag_id, ) )
pending_delta = self._GetRowCount()
counts_cache_changes.append( ( storage_tag_id, 0, pending_delta ) )
self.modules_mappings_counts_update.AddCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )
self.modules_mappings_cache_specific_display.RegeneratePending( file_service_id, tag_service_id, status_hook = status_hook )
def AddFiles( self, file_service_id, tag_service_id, hash_ids, hash_ids_table_name ):
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( tag_service_id )
# deleted don't have a/c counts to update, so we can do it all in one go here
self._Execute( 'INSERT OR IGNORE INTO {} ( hash_id, tag_id ) SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( cache_deleted_mappings_table_name, hash_ids_table_name, deleted_mappings_table_name ) )
# temp hashes to mappings
current_mapping_ids_raw = self._Execute( 'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( hash_ids_table_name, current_mappings_table_name ) ).fetchall()
current_mapping_ids_dict = HydrusData.BuildKeyToSetDict( current_mapping_ids_raw )
# temp hashes to mappings
pending_mapping_ids_raw = self._Execute( 'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( hash_ids_table_name, pending_mappings_table_name ) ).fetchall()
pending_mapping_ids_dict = HydrusData.BuildKeyToSetDict( pending_mapping_ids_raw )
all_ids_seen = set( current_mapping_ids_dict.keys() )
all_ids_seen.update( pending_mapping_ids_dict.keys() )
counts_cache_changes = []
for tag_id in all_ids_seen:
current_hash_ids = current_mapping_ids_dict[ tag_id ]
current_delta = len( current_hash_ids )
if current_delta > 0:
self._ExecuteMany( 'INSERT OR IGNORE INTO ' + cache_current_mappings_table_name + ' ( hash_id, tag_id ) VALUES ( ?, ? );', ( ( hash_id, tag_id ) for hash_id in current_hash_ids ) )
current_delta = self._GetRowCount()
#
pending_hash_ids = pending_mapping_ids_dict[ tag_id ]
pending_delta = len( pending_hash_ids )
if pending_delta > 0:
self._ExecuteMany( 'INSERT OR IGNORE INTO ' + cache_pending_mappings_table_name + ' ( hash_id, tag_id ) VALUES ( ?, ? );', ( ( hash_id, tag_id ) for hash_id in pending_hash_ids ) )
pending_delta = self._GetRowCount()
#
if current_delta > 0 or pending_delta > 0:
counts_cache_changes.append( ( tag_id, current_delta, pending_delta ) )
if len( counts_cache_changes ) > 0:
self.modules_mappings_counts_update.AddCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )