def Generate(self,
file_service_id,
tag_service_id,
populate_from_storage=True,
status_hook=None):
table_generation_dict = self._GetServiceTableGenerationDictSingle(
file_service_id, tag_service_id)
for (table_name, (create_query_without_name,
version_added)) in table_generation_dict.items():
self._Execute(create_query_without_name.format(table_name))
if populate_from_storage:
if status_hook is not None:
status_hook('copying storage')
(
cache_current_mappings_table_name,
cache_deleted_mappings_table_name,
cache_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames(
file_service_id, tag_service_id)
(
cache_display_current_mappings_table_name,
cache_display_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificDisplayMappingsCacheTableNames(
file_service_id, tag_service_id)
self._Execute(
'INSERT OR IGNORE INTO {} ( hash_id, tag_id ) SELECT hash_id, tag_id FROM {};'
.format(cache_display_current_mappings_table_name,
cache_current_mappings_table_name))
self._Execute(
'INSERT OR IGNORE INTO {} ( hash_id, tag_id ) SELECT hash_id, tag_id FROM {};'
.format(cache_display_pending_mappings_table_name,
cache_pending_mappings_table_name))
self.modules_mappings_counts.CreateTables(
ClientTags.TAG_DISPLAY_ACTUAL,
file_service_id,
tag_service_id,
populate_from_storage=populate_from_storage)
if status_hook is not None:
status_hook('optimising data')
index_generation_dict = self._GetServiceIndexGenerationDictSingle(
file_service_id, tag_service_id)
for (table_name, columns, unique, version_added
) in self._FlattenIndexGenerationDict(index_generation_dict):
self._CreateIndex(table_name, columns, unique=unique)
def RegeneratePending(self,
file_service_id,
tag_service_id,
status_hook=None):
(cache_current_mappings_table_name, cache_deleted_mappings_table_name,
cache_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames(
file_service_id, tag_service_id)
(
cache_display_current_mappings_table_name,
cache_display_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificDisplayMappingsCacheTableNames(
file_service_id, tag_service_id)
if status_hook is not None:
message = 'clearing old specific display data'
status_hook(message)
all_pending_storage_tag_ids = self._STS(
self._Execute('SELECT DISTINCT tag_id FROM {};'.format(
cache_pending_mappings_table_name)))
storage_tag_ids_to_display_tag_ids = self.modules_tag_display.GetTagsToImplies(
ClientTags.TAG_DISPLAY_ACTUAL, tag_service_id,
all_pending_storage_tag_ids)
all_pending_display_tag_ids = set(
itertools.chain.from_iterable(
storage_tag_ids_to_display_tag_ids.values()))
del all_pending_storage_tag_ids
del storage_tag_ids_to_display_tag_ids
self.modules_mappings_counts.ClearCounts(ClientTags.TAG_DISPLAY_ACTUAL,
file_service_id,
tag_service_id,
keep_current=True)
self._Execute('DELETE FROM {};'.format(
cache_display_pending_mappings_table_name))
all_pending_display_tag_ids_to_implied_by_storage_tag_ids = self.modules_tag_display.GetTagsToImpliedBy(
ClientTags.TAG_DISPLAY_ACTUAL,
tag_service_id,
all_pending_display_tag_ids,
tags_are_ideal=True)
counts_cache_changes = []
num_to_do = len(
all_pending_display_tag_ids_to_implied_by_storage_tag_ids)
for (i, (display_tag_id, storage_tag_ids)) in enumerate(
all_pending_display_tag_ids_to_implied_by_storage_tag_ids.
items()):
if i % 100 == 0 and status_hook is not None:
message = 'regenerating pending tags {}'.format(
HydrusData.ConvertValueRangeToPrettyString(
i + 1, num_to_do))
status_hook(message)
if len(storage_tag_ids) == 1:
(storage_tag_id, ) = storage_tag_ids
self._Execute(
'INSERT OR IGNORE INTO {} ( tag_id, hash_id ) SELECT ?, hash_id FROM {} WHERE tag_id = ?;'
.format(cache_display_pending_mappings_table_name,
cache_pending_mappings_table_name),
(display_tag_id, storage_tag_id))
pending_delta = self._GetRowCount()
else:
with self._MakeTemporaryIntegerTable(
storage_tag_ids, 'tag_id') as temp_tag_ids_table_name:
# temp tags to mappings merged
self._Execute(
'INSERT OR IGNORE INTO {} ( tag_id, hash_id ) SELECT DISTINCT ?, hash_id FROM {} CROSS JOIN {} USING ( tag_id );'
.format(cache_display_pending_mappings_table_name,
temp_tag_ids_table_name,
cache_pending_mappings_table_name),
(display_tag_id, ))
pending_delta = self._GetRowCount()
counts_cache_changes.append((display_tag_id, 0, pending_delta))
self.modules_mappings_counts_update.AddCounts(
ClientTags.TAG_DISPLAY_ACTUAL, file_service_id, tag_service_id,
counts_cache_changes)
def AddImplications(self,
file_service_id,
tag_service_id,
implication_tag_ids,
tag_id,
status_hook=None):
if len(implication_tag_ids) == 0:
return
(cache_current_mappings_table_name, cache_deleted_mappings_table_name,
cache_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames(
file_service_id, tag_service_id)
(
cache_display_current_mappings_table_name,
cache_display_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificDisplayMappingsCacheTableNames(
file_service_id, tag_service_id)
statuses_to_count_delta = collections.Counter()
(
current_implication_tag_ids, current_implication_tag_ids_weight,
pending_implication_tag_ids, pending_implication_tag_ids_weight
) = self.modules_mappings_counts.GetCurrentPendingPositiveCountsAndWeights(
ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id,
implication_tag_ids)
jobs = []
jobs.append(
(HC.CONTENT_STATUS_CURRENT,
cache_display_current_mappings_table_name,
cache_current_mappings_table_name, current_implication_tag_ids,
current_implication_tag_ids_weight))
jobs.append(
(HC.CONTENT_STATUS_PENDING,
cache_display_pending_mappings_table_name,
cache_pending_mappings_table_name, pending_implication_tag_ids,
pending_implication_tag_ids_weight))
for (status, cache_display_mappings_table_name,
cache_mappings_table_name, add_tag_ids,
add_tag_ids_weight) in jobs:
if add_tag_ids_weight == 0:
# nothing to actually add, so nbd
continue
if len(add_tag_ids) == 1:
(add_tag_id, ) = add_tag_ids
self._Execute(
'INSERT OR IGNORE INTO {} ( hash_id, tag_id ) SELECT hash_id, ? FROM {} WHERE tag_id = ?;'
.format(cache_display_mappings_table_name,
cache_mappings_table_name), (tag_id, add_tag_id))
statuses_to_count_delta[status] = self._GetRowCount()
else:
with self._MakeTemporaryIntegerTable(
add_tag_ids, 'tag_id') as temp_tag_ids_table_name:
# for all new implications, get files with those tags and not existing
self._Execute(
'INSERT OR IGNORE INTO {} ( hash_id, tag_id ) SELECT hash_id, ? FROM {} CROSS JOIN {} USING ( tag_id );'
.format(cache_display_mappings_table_name,
temp_tag_ids_table_name,
cache_mappings_table_name), (tag_id, ))
statuses_to_count_delta[status] = self._GetRowCount()
current_delta = statuses_to_count_delta[HC.CONTENT_STATUS_CURRENT]
pending_delta = statuses_to_count_delta[HC.CONTENT_STATUS_PENDING]
if current_delta > 0 or pending_delta > 0:
counts_cache_changes = ((tag_id, current_delta, pending_delta), )
self.modules_mappings_counts_update.AddCounts(
ClientTags.TAG_DISPLAY_ACTUAL, file_service_id, tag_service_id,
counts_cache_changes)
def DeleteImplications(self,
file_service_id,
tag_service_id,
implication_tag_ids,
tag_id,
status_hook=None):
if len(implication_tag_ids) == 0:
return
statuses_to_count_delta = collections.Counter()
(cache_current_mappings_table_name, cache_deleted_mappings_table_name,
cache_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames(
file_service_id, tag_service_id)
(
cache_display_current_mappings_table_name,
cache_display_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificDisplayMappingsCacheTableNames(
file_service_id, tag_service_id)
remaining_implication_tag_ids = set(
self.modules_tag_display.GetImpliedBy(
ClientTags.TAG_DISPLAY_ACTUAL, tag_service_id,
tag_id)).difference(implication_tag_ids)
(
current_implication_tag_ids, current_implication_tag_ids_weight,
pending_implication_tag_ids, pending_implication_tag_ids_weight
) = self.modules_mappings_counts.GetCurrentPendingPositiveCountsAndWeights(
ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id,
implication_tag_ids)
(
current_remaining_implication_tag_ids,
current_remaining_implication_tag_ids_weight,
pending_remaining_implication_tag_ids,
pending_remaining_implication_tag_ids_weight
) = self.modules_mappings_counts.GetCurrentPendingPositiveCountsAndWeights(
ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id,
remaining_implication_tag_ids)
jobs = []
jobs.append(
(HC.CONTENT_STATUS_CURRENT,
cache_display_current_mappings_table_name,
cache_current_mappings_table_name, current_implication_tag_ids,
current_implication_tag_ids_weight,
current_remaining_implication_tag_ids,
current_remaining_implication_tag_ids_weight))
jobs.append(
(HC.CONTENT_STATUS_PENDING,
cache_display_pending_mappings_table_name,
cache_pending_mappings_table_name, pending_implication_tag_ids,
pending_implication_tag_ids_weight,
pending_remaining_implication_tag_ids,
pending_remaining_implication_tag_ids_weight))
for (status, cache_display_mappings_table_name,
cache_mappings_table_name, removee_tag_ids,
removee_tag_ids_weight, keep_tag_ids,
keep_tag_ids_weight) in jobs:
if removee_tag_ids_weight == 0:
# nothing to remove, so nothing to do!
continue
# ultimately here, we are doing "delete all display mappings with hash_ids that have a storage mapping for a removee tag and no storage mappings for a keep tag
# in order to reduce overhead, we go full meme and do a bunch of different situations
with self._MakeTemporaryIntegerTable(
[], 'tag_id') as temp_removee_tag_ids_table_name:
with self._MakeTemporaryIntegerTable(
[], 'tag_id') as temp_keep_tag_ids_table_name:
if len(removee_tag_ids) == 1:
(removee_tag_id, ) = removee_tag_ids
hash_id_in_storage_remove = 'hash_id IN ( SELECT hash_id FROM {} WHERE tag_id = {} )'.format(
cache_mappings_table_name, removee_tag_id)
else:
self._ExecuteMany(
'INSERT INTO {} ( tag_id ) VALUES ( ? );'.format(
temp_removee_tag_ids_table_name),
((removee_tag_id, )
for removee_tag_id in removee_tag_ids))
hash_id_in_storage_remove = 'hash_id IN ( SELECT DISTINCT hash_id FROM {} CROSS JOIN {} USING ( tag_id ) )'.format(
temp_removee_tag_ids_table_name,
cache_mappings_table_name)
if keep_tag_ids_weight == 0:
predicates_phrase = hash_id_in_storage_remove
else:
# WARNING, WARNING: Big Brain Query, potentially great/awful
# note that in the 'clever/file join' situation, the number of total mappings is many, but we are deleting a few
# we want to precisely scan the status of the potential hashes to delete, not scan through them all to see what not to do
# therefore, we do NOT EXISTS, which just scans the parts, rather than NOT IN, which does the whole query and then checks against all results
if len(keep_tag_ids) == 1:
(keep_tag_id, ) = keep_tag_ids
if ClientDBMappingsStorage.DoingAFileJoinTagSearchIsFaster(
removee_tag_ids_weight,
keep_tag_ids_weight):
hash_id_not_in_storage_keep = 'NOT EXISTS ( SELECT 1 FROM {} WHERE {}.hash_id = {}.hash_id and tag_id = {} )'.format(
cache_mappings_table_name,
cache_display_mappings_table_name,
cache_mappings_table_name, keep_tag_id)
else:
hash_id_not_in_storage_keep = 'hash_id NOT IN ( SELECT hash_id FROM {} WHERE tag_id = {} )'.format(
cache_mappings_table_name, keep_tag_id)
else:
self._ExecuteMany(
'INSERT INTO {} ( tag_id ) VALUES ( ? );'.
format(temp_keep_tag_ids_table_name),
((keep_tag_id, )
for keep_tag_id in keep_tag_ids))
if ClientDBMappingsStorage.DoingAFileJoinTagSearchIsFaster(
removee_tag_ids_weight,
keep_tag_ids_weight):
# (files to) mappings to temp tags
hash_id_not_in_storage_keep = 'NOT EXISTS ( SELECT 1 FROM {} CROSS JOIN {} USING ( tag_id ) WHERE {}.hash_id = {}.hash_id )'.format(
cache_mappings_table_name,
temp_keep_tag_ids_table_name,
cache_display_mappings_table_name,
cache_mappings_table_name)
else:
# temp tags to mappings
hash_id_not_in_storage_keep = ' hash_id NOT IN ( SELECT DISTINCT hash_id FROM {} CROSS JOIN {} USING ( tag_id ) )'.format(
temp_keep_tag_ids_table_name,
cache_mappings_table_name)
predicates_phrase = '{} AND {}'.format(
hash_id_in_storage_remove,
hash_id_not_in_storage_keep)
query = 'DELETE FROM {} WHERE tag_id = {} AND {};'.format(
cache_display_mappings_table_name, tag_id,
predicates_phrase)
self._Execute(query)
statuses_to_count_delta[status] = self._GetRowCount()
current_delta = statuses_to_count_delta[HC.CONTENT_STATUS_CURRENT]
pending_delta = statuses_to_count_delta[HC.CONTENT_STATUS_PENDING]
if current_delta > 0 or pending_delta > 0:
counts_cache_changes = ((tag_id, current_delta, pending_delta), )
self.modules_mappings_counts_update.ReduceCounts(
ClientTags.TAG_DISPLAY_ACTUAL, file_service_id, tag_service_id,
counts_cache_changes)
def _GetServiceIndexGenerationDictSingle( self, file_service_id, tag_service_id ):
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
version = 486 if file_service_id == self.modules_services.combined_local_media_service_id else 400
index_generation_dict = {}
index_generation_dict[ cache_current_mappings_table_name ] = [
( [ 'tag_id', 'hash_id' ], True, version )
]
index_generation_dict[ cache_deleted_mappings_table_name ] = [
( [ 'tag_id', 'hash_id' ], True, version )
]
index_generation_dict[ cache_pending_mappings_table_name ] = [
( [ 'tag_id', 'hash_id' ], True, version )
]
return index_generation_dict
def AddFiles(self, file_service_id, tag_service_id, hash_ids,
hash_ids_table_name):
(
cache_display_current_mappings_table_name,
cache_display_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificDisplayMappingsCacheTableNames(
file_service_id, tag_service_id)
(cache_current_mappings_table_name, cache_deleted_mappings_table_name,
cache_pending_mappings_table_name
) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames(
file_service_id, tag_service_id)
# temp hashes to mappings
storage_current_mapping_ids_raw = self._Execute(
'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.
format(hash_ids_table_name,
cache_current_mappings_table_name)).fetchall()
storage_current_mapping_ids_dict = HydrusData.BuildKeyToSetDict(
storage_current_mapping_ids_raw)
# temp hashes to mappings
storage_pending_mapping_ids_raw = self._Execute(
'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.
format(hash_ids_table_name,
cache_pending_mappings_table_name)).fetchall()
storage_pending_mapping_ids_dict = HydrusData.BuildKeyToSetDict(
storage_pending_mapping_ids_raw)
all_storage_tag_ids = set(storage_current_mapping_ids_dict.keys())
all_storage_tag_ids.update(storage_pending_mapping_ids_dict.keys())
storage_tag_ids_to_implies_tag_ids = self.modules_tag_display.GetTagsToImplies(
ClientTags.TAG_DISPLAY_ACTUAL, tag_service_id, all_storage_tag_ids)
display_tag_ids_to_implied_by_tag_ids = collections.defaultdict(set)
for (storage_tag_id,
implies_tag_ids) in storage_tag_ids_to_implies_tag_ids.items():
for implies_tag_id in implies_tag_ids:
display_tag_ids_to_implied_by_tag_ids[implies_tag_id].add(
storage_tag_id)
counts_cache_changes = []
# for all display tags implied by the existing storage mappings, add them
# btw, when we add files to a specific domain, we know that all inserts are new
for (display_tag_id, implied_by_tag_ids
) in display_tag_ids_to_implied_by_tag_ids.items():
display_current_hash_ids = set(
itertools.chain.from_iterable(
(storage_current_mapping_ids_dict[implied_by_tag_id]
for implied_by_tag_id in implied_by_tag_ids)))
current_delta = len(display_current_hash_ids)
if current_delta > 0:
self._ExecuteMany(
'INSERT OR IGNORE INTO ' +
cache_display_current_mappings_table_name +
' ( hash_id, tag_id ) VALUES ( ?, ? );',
((hash_id, display_tag_id)
for hash_id in display_current_hash_ids))
#
display_pending_hash_ids = set(
itertools.chain.from_iterable(
(storage_pending_mapping_ids_dict[implied_by_tag_id]
for implied_by_tag_id in implied_by_tag_ids)))
pending_delta = len(display_pending_hash_ids)
if pending_delta > 0:
self._ExecuteMany(
'INSERT OR IGNORE INTO ' +
cache_display_pending_mappings_table_name +
' ( hash_id, tag_id ) VALUES ( ?, ? );',
((hash_id, display_tag_id)
for hash_id in display_pending_hash_ids))
#
if current_delta > 0 or pending_delta > 0:
counts_cache_changes.append(
(display_tag_id, current_delta, pending_delta))
if len(counts_cache_changes) > 0:
self.modules_mappings_counts_update.AddCounts(
ClientTags.TAG_DISPLAY_ACTUAL, file_service_id, tag_service_id,
counts_cache_changes)
def RegeneratePending( self, file_service_id, tag_service_id, status_hook = None ):
( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( tag_service_id )
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
if status_hook is not None:
message = 'clearing old specific data'
status_hook( message )
all_pending_storage_tag_ids = self._STS( self._Execute( 'SELECT DISTINCT tag_id FROM {};'.format( pending_mappings_table_name ) ) )
self.modules_mappings_counts.ClearCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, keep_current = True )
self._Execute( 'DELETE FROM {};'.format( cache_pending_mappings_table_name ) )
counts_cache_changes = []
num_to_do = len( all_pending_storage_tag_ids )
select_table_join = self.modules_files_storage.GetTableJoinLimitedByFileDomain( file_service_id, pending_mappings_table_name, HC.CONTENT_STATUS_CURRENT )
for ( i, storage_tag_id ) in enumerate( all_pending_storage_tag_ids ):
if i % 100 == 0 and status_hook is not None:
message = 'regenerating pending tags {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, num_to_do ) )
status_hook( message )
self._Execute( 'INSERT OR IGNORE INTO {} ( tag_id, hash_id ) SELECT tag_id, hash_id FROM {} WHERE tag_id = ?;'.format( cache_pending_mappings_table_name, select_table_join ), ( storage_tag_id, ) )
pending_delta = self._GetRowCount()
counts_cache_changes.append( ( storage_tag_id, 0, pending_delta ) )
self.modules_mappings_counts_update.AddCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )
self.modules_mappings_cache_specific_display.RegeneratePending( file_service_id, tag_service_id, status_hook = status_hook )
def RescindPendingMappings( self, tag_service_id, tag_id, hash_ids, filtered_hashes_generator: FilteredHashesGenerator ):
for ( file_service_id, filtered_hash_ids ) in filtered_hashes_generator.IterateHashes( hash_ids ):
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
ac_counts = collections.Counter()
self.modules_mappings_cache_specific_display.RescindPendingMappings( file_service_id, tag_service_id, tag_id, filtered_hash_ids )
self._ExecuteMany( 'DELETE FROM ' + cache_pending_mappings_table_name + ' WHERE hash_id = ? AND tag_id = ?;', ( ( hash_id, tag_id ) for hash_id in filtered_hash_ids ) )
num_deleted = self._GetRowCount()
if num_deleted > 0:
counts_cache_changes = [ ( tag_id, 0, num_deleted ) ]
self.modules_mappings_counts_update.ReduceCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )
def DeleteFiles( self, file_service_id, tag_service_id, hash_ids, hash_id_table_name ):
self.modules_mappings_cache_specific_display.DeleteFiles( file_service_id, tag_service_id, hash_ids, hash_id_table_name )
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
# temp hashes to mappings
deleted_mapping_ids_raw = self._Execute( 'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( hash_id_table_name, cache_deleted_mappings_table_name ) ).fetchall()
if len( deleted_mapping_ids_raw ) > 0:
self._ExecuteMany( 'DELETE FROM {} WHERE tag_id = ? AND hash_id = ?;'.format( cache_deleted_mappings_table_name ), deleted_mapping_ids_raw )
# temp hashes to mappings
current_mapping_ids_raw = self._Execute( 'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( hash_id_table_name, cache_current_mappings_table_name ) ).fetchall()
current_mapping_ids_dict = HydrusData.BuildKeyToSetDict( current_mapping_ids_raw )
# temp hashes to mappings
pending_mapping_ids_raw = self._Execute( 'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( hash_id_table_name, cache_pending_mappings_table_name ) ).fetchall()
pending_mapping_ids_dict = HydrusData.BuildKeyToSetDict( pending_mapping_ids_raw )
all_ids_seen = set( current_mapping_ids_dict.keys() )
all_ids_seen.update( pending_mapping_ids_dict.keys() )
counts_cache_changes = []
for tag_id in all_ids_seen:
current_hash_ids = current_mapping_ids_dict[ tag_id ]
num_current = len( current_hash_ids )
#
pending_hash_ids = pending_mapping_ids_dict[ tag_id ]
num_pending = len( pending_hash_ids )
counts_cache_changes.append( ( tag_id, num_current, num_pending ) )
self._ExecuteMany( 'DELETE FROM ' + cache_current_mappings_table_name + ' WHERE hash_id = ?;', ( ( hash_id, ) for hash_id in hash_ids ) )
self._ExecuteMany( 'DELETE FROM ' + cache_pending_mappings_table_name + ' WHERE hash_id = ?;', ( ( hash_id, ) for hash_id in hash_ids ) )
if len( counts_cache_changes ) > 0:
self.modules_mappings_counts_update.ReduceCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )
def PendMappings( self, tag_service_id, tag_id, hash_ids, filtered_hashes_generator: FilteredHashesGenerator ):
for ( file_service_id, filtered_hash_ids ) in filtered_hashes_generator.IterateHashes( hash_ids ):
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
self._ExecuteMany( 'INSERT OR IGNORE INTO ' + cache_pending_mappings_table_name + ' ( hash_id, tag_id ) VALUES ( ?, ? );', ( ( hash_id, tag_id ) for hash_id in filtered_hash_ids ) )
num_added = self._GetRowCount()
if num_added > 0:
counts_cache_changes = [ ( tag_id, 0, num_added ) ]
self.modules_mappings_counts_update.AddCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )
self.modules_mappings_cache_specific_display.PendMappings( file_service_id, tag_service_id, tag_id, filtered_hash_ids )
def Drop( self, file_service_id, tag_service_id ):
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
self._Execute( 'DROP TABLE IF EXISTS {};'.format( cache_current_mappings_table_name ) )
self._Execute( 'DROP TABLE IF EXISTS {};'.format( cache_deleted_mappings_table_name ) )
self._Execute( 'DROP TABLE IF EXISTS {};'.format( cache_pending_mappings_table_name ) )
self.modules_mappings_counts.DropTables( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id )
self.modules_mappings_cache_specific_display.Drop( file_service_id, tag_service_id )
def Clear( self, file_service_id, tag_service_id, keep_pending = False ):
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
self._Execute( 'DELETE FROM {};'.format( cache_current_mappings_table_name ) )
self._Execute( 'DELETE FROM {};'.format( cache_deleted_mappings_table_name ) )
if not keep_pending:
self._Execute( 'DELETE FROM {};'.format( cache_pending_mappings_table_name ) )
self.modules_mappings_counts.ClearCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, keep_pending = keep_pending )
self.modules_mappings_cache_specific_display.Clear( file_service_id, tag_service_id, keep_pending = keep_pending )
def AddMappings( self, tag_service_id, tag_id, hash_ids, filtered_hashes_generator: FilteredHashesGenerator ):
for ( file_service_id, filtered_hash_ids ) in filtered_hashes_generator.IterateHashes( hash_ids ):
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
# we have to interleave this into the iterator so that if two siblings with the same ideal are pend->currented at once, we remain logic consistent for soletag lookups!
self.modules_mappings_cache_specific_display.RescindPendingMappings( file_service_id, tag_service_id, tag_id, filtered_hash_ids )
self._ExecuteMany( 'DELETE FROM ' + cache_pending_mappings_table_name + ' WHERE hash_id = ? AND tag_id = ?;', ( ( hash_id, tag_id ) for hash_id in filtered_hash_ids ) )
num_pending_rescinded = self._GetRowCount()
#
self._ExecuteMany( 'INSERT OR IGNORE INTO ' + cache_current_mappings_table_name + ' ( hash_id, tag_id ) VALUES ( ?, ? );', ( ( hash_id, tag_id ) for hash_id in filtered_hash_ids ) )
num_current_inserted = self._GetRowCount()
#
self._ExecuteMany( 'DELETE FROM ' + cache_deleted_mappings_table_name + ' WHERE hash_id = ? AND tag_id = ?;', ( ( hash_id, tag_id ) for hash_id in filtered_hash_ids ) )
if num_current_inserted > 0:
counts_cache_changes = [ ( tag_id, num_current_inserted, 0 ) ]
self.modules_mappings_counts_update.AddCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )
if num_pending_rescinded > 0:
counts_cache_changes = [ ( tag_id, 0, num_pending_rescinded ) ]
self.modules_mappings_counts_update.ReduceCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )
self.modules_mappings_cache_specific_display.AddMappings( file_service_id, tag_service_id, tag_id, filtered_hash_ids )
def AddFiles( self, file_service_id, tag_service_id, hash_ids, hash_ids_table_name ):
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( tag_service_id )
# deleted don't have a/c counts to update, so we can do it all in one go here
self._Execute( 'INSERT OR IGNORE INTO {} ( hash_id, tag_id ) SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( cache_deleted_mappings_table_name, hash_ids_table_name, deleted_mappings_table_name ) )
# temp hashes to mappings
current_mapping_ids_raw = self._Execute( 'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( hash_ids_table_name, current_mappings_table_name ) ).fetchall()
current_mapping_ids_dict = HydrusData.BuildKeyToSetDict( current_mapping_ids_raw )
# temp hashes to mappings
pending_mapping_ids_raw = self._Execute( 'SELECT tag_id, hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( hash_ids_table_name, pending_mappings_table_name ) ).fetchall()
pending_mapping_ids_dict = HydrusData.BuildKeyToSetDict( pending_mapping_ids_raw )
all_ids_seen = set( current_mapping_ids_dict.keys() )
all_ids_seen.update( pending_mapping_ids_dict.keys() )
counts_cache_changes = []
for tag_id in all_ids_seen:
current_hash_ids = current_mapping_ids_dict[ tag_id ]
current_delta = len( current_hash_ids )
if current_delta > 0:
self._ExecuteMany( 'INSERT OR IGNORE INTO ' + cache_current_mappings_table_name + ' ( hash_id, tag_id ) VALUES ( ?, ? );', ( ( hash_id, tag_id ) for hash_id in current_hash_ids ) )
current_delta = self._GetRowCount()
#
pending_hash_ids = pending_mapping_ids_dict[ tag_id ]
pending_delta = len( pending_hash_ids )
if pending_delta > 0:
self._ExecuteMany( 'INSERT OR IGNORE INTO ' + cache_pending_mappings_table_name + ' ( hash_id, tag_id ) VALUES ( ?, ? );', ( ( hash_id, tag_id ) for hash_id in pending_hash_ids ) )
pending_delta = self._GetRowCount()
#
if current_delta > 0 or pending_delta > 0:
counts_cache_changes.append( ( tag_id, current_delta, pending_delta ) )
if len( counts_cache_changes ) > 0:
self.modules_mappings_counts_update.AddCounts( ClientTags.TAG_DISPLAY_STORAGE, file_service_id, tag_service_id, counts_cache_changes )
def _GetServiceTableGenerationDictSingle( self, file_service_id, tag_service_id ):
table_dict = {}
( cache_current_mappings_table_name, cache_deleted_mappings_table_name, cache_pending_mappings_table_name ) = ClientDBMappingsStorage.GenerateSpecificMappingsCacheTableNames( file_service_id, tag_service_id )
version = 486 if file_service_id == self.modules_services.combined_local_media_service_id else 400
table_dict[ cache_current_mappings_table_name ] = ( 'CREATE TABLE IF NOT EXISTS {} ( hash_id INTEGER, tag_id INTEGER, PRIMARY KEY ( hash_id, tag_id ) ) WITHOUT ROWID;', version )
table_dict[ cache_deleted_mappings_table_name ] = ( 'CREATE TABLE IF NOT EXISTS {} ( hash_id INTEGER, tag_id INTEGER, PRIMARY KEY ( hash_id, tag_id ) ) WITHOUT ROWID;', version )
table_dict[ cache_pending_mappings_table_name ] = ( 'CREATE TABLE IF NOT EXISTS {} ( hash_id INTEGER, tag_id INTEGER, PRIMARY KEY ( hash_id, tag_id ) ) WITHOUT ROWID;', version )
return table_dict