def DoWork( self ):
if HG.view_shutdown:
return
if HC.options[ 'pause_import_folders_sync' ] or self._paused:
return
checked_folder = False
did_import_file_work = False
error_occured = False
stop_time = HydrusData.GetNow() + 3600
job_key = ClientThreading.JobKey( pausable = False, cancellable = True, stop_time = stop_time )
try:
if not os.path.exists( self._path ) or not os.path.isdir( self._path ):
raise Exception( 'Path "' + self._path + '" does not seem to exist, or is not a directory.' )
pubbed_job_key = False
job_key.SetStatusTitle( 'import folder - ' + self._name )
due_by_check_now = self._check_now
due_by_period = self._check_regularly and HydrusData.TimeHasPassed( self._last_checked + self._period )
if due_by_check_now or due_by_period:
if not pubbed_job_key and self._show_working_popup:
HG.client_controller.pub( 'message', job_key )
pubbed_job_key = True
self._CheckFolder( job_key )
checked_folder = True
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
if file_seed is not None:
if not pubbed_job_key and self._show_working_popup:
HG.client_controller.pub( 'message', job_key )
pubbed_job_key = True
did_import_file_work = self._ImportFiles( job_key )
except Exception as e:
error_occured = True
self._paused = True
HydrusData.ShowText( 'The import folder "' + self._name + '" encountered an exception! It has been paused!' )
HydrusData.ShowException( e )
if checked_folder or did_import_file_work or error_occured:
HG.client_controller.WriteSynchronous( 'serialisable', self )
job_key.Delete()
def Search(self, hash_id, max_hamming_distance):
if max_hamming_distance == 0:
similar_hash_ids = self._STL(
self._c.execute(
'SELECT hash_id FROM shape_perceptual_hash_map WHERE phash_id IN ( SELECT phash_id FROM shape_perceptual_hash_map WHERE hash_id = ? );',
(hash_id, )))
similar_hash_ids_and_distances = [
(similar_hash_id, 0) for similar_hash_id in similar_hash_ids
]
else:
search_radius = max_hamming_distance
top_node_result = self._c.execute(
'SELECT phash_id FROM shape_vptree WHERE parent_id IS NULL;'
).fetchone()
if top_node_result is None:
return []
(root_node_phash_id, ) = top_node_result
search = self._STL(
self._c.execute(
'SELECT phash FROM shape_perceptual_hashes NATURAL JOIN shape_perceptual_hash_map WHERE hash_id = ?;',
(hash_id, )))
if len(search) == 0:
return []
similar_phash_ids_to_distances = {}
num_cycles = 0
total_nodes_searched = 0
for search_phash in search:
next_potentials = [root_node_phash_id]
while len(next_potentials) > 0:
current_potentials = next_potentials
next_potentials = []
num_cycles += 1
total_nodes_searched += len(current_potentials)
for group_of_current_potentials in HydrusData.SplitListIntoChunks(
current_potentials, 10000):
# this is split into fixed lists of results of subgroups because as an iterable it was causing crashes on linux!!
# after investigation, it seemed to be SQLite having a problem with part of Get64BitHammingDistance touching phashes it presumably was still hanging on to
# the crash was in sqlite code, again presumably on subsequent fetch
# adding a delay in seemed to fix it as well. guess it was some memory maintenance buffer/bytes thing
# anyway, we now just get the whole lot of results first and then work on the whole lot
'''
#old method
select_statement = 'SELECT phash_id, phash, radius, inner_id, outer_id FROM shape_perceptual_hashes NATURAL JOIN shape_vptree WHERE phash_id = ?;'
results = list( self._ExecuteManySelectSingleParam( select_statement, group_of_current_potentials ) )
'''
with HydrusDB.TemporaryIntegerTable(
self._c, group_of_current_potentials,
'phash_id') as temp_table_name:
# temp phash_ids to actual phashes and tree info
results = self._c.execute(
'SELECT phash_id, phash, radius, inner_id, outer_id FROM {} CROSS JOIN shape_perceptual_hashes USING ( phash_id ) CROSS JOIN shape_vptree USING ( phash_id );'
.format(temp_table_name)).fetchall()
for (node_phash_id, node_phash, node_radius,
inner_phash_id, outer_phash_id) in results:
# first check the node itself--is it similar?
node_hamming_distance = HydrusData.Get64BitHammingDistance(
search_phash, node_phash)
if node_hamming_distance <= search_radius:
if node_phash_id in similar_phash_ids_to_distances:
current_distance = similar_phash_ids_to_distances[
node_phash_id]
similar_phash_ids_to_distances[
node_phash_id] = min(
node_hamming_distance,
current_distance)
else:
similar_phash_ids_to_distances[
node_phash_id] = node_hamming_distance
# now how about its children?
if node_radius is not None:
# we have two spheres--node and search--their centers separated by node_hamming_distance
# we want to search inside/outside the node_sphere if the search_sphere intersects with those spaces
# there are four possibles:
# (----N----)-(--S--) intersects with outer only - distance between N and S > their radii
# (----N---(-)-S--) intersects with both
# (----N-(--S-)-) intersects with both
# (---(-N-S--)-) intersects with inner only - distance between N and S + radius_S does not exceed radius_N
if inner_phash_id is not None:
spheres_disjoint = node_hamming_distance > (
node_radius + search_radius)
if not spheres_disjoint: # i.e. they intersect at some point
next_potentials.append(inner_phash_id)
if outer_phash_id is not None:
search_sphere_subset_of_node_sphere = (
node_hamming_distance +
search_radius) <= node_radius
if not search_sphere_subset_of_node_sphere: # i.e. search sphere intersects with non-node sphere space at some point
next_potentials.append(outer_phash_id)
if HG.db_report_mode:
HydrusData.ShowText(
'Similar file search touched {} nodes over {} cycles.'.
format(HydrusData.ToHumanInt(total_nodes_searched),
HydrusData.ToHumanInt(num_cycles)))
# so, now we have phash_ids and distances. let's map that to actual files.
# files can have multiple phashes, and phashes can refer to multiple files, so let's make sure we are setting the smallest distance we found
similar_phash_ids = list(similar_phash_ids_to_distances.keys())
with HydrusDB.TemporaryIntegerTable(self._c, similar_phash_ids,
'phash_id') as temp_table_name:
# temp phashes to hash map
similar_phash_ids_to_hash_ids = HydrusData.BuildKeyToListDict(
self._c.execute(
'SELECT phash_id, hash_id FROM {} CROSS JOIN shape_perceptual_hash_map USING ( phash_id );'
.format(temp_table_name)))
similar_hash_ids_to_distances = {}
for (phash_id, hash_ids) in similar_phash_ids_to_hash_ids.items():
distance = similar_phash_ids_to_distances[phash_id]
for hash_id in hash_ids:
if hash_id not in similar_hash_ids_to_distances:
similar_hash_ids_to_distances[hash_id] = distance
else:
current_distance = similar_hash_ids_to_distances[
hash_id]
if distance < current_distance:
similar_hash_ids_to_distances[hash_id] = distance
similar_hash_ids_and_distances = list(
similar_hash_ids_to_distances.items())
return similar_hash_ids_and_distances
def _ActionPaths(self):
for status in (CC.STATUS_SUCCESSFUL_AND_NEW,
CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, CC.STATUS_DELETED,
CC.STATUS_ERROR):
action = self._actions[status]
if action == CC.IMPORT_FOLDER_DELETE:
while True:
file_seed = self._file_seed_cache.GetNextFileSeed(status)
if file_seed is None or HG.view_shutdown:
break
path = file_seed.file_seed_data
try:
if os.path.exists(path) and not os.path.isdir(path):
ClientPaths.DeletePath(path)
txt_path = path + '.txt'
if os.path.exists(txt_path):
ClientPaths.DeletePath(txt_path)
self._file_seed_cache.RemoveFileSeeds((file_seed, ))
except Exception as e:
raise Exception(
'Tried to delete "{}", but could not.'.format(
path))
elif action == CC.IMPORT_FOLDER_MOVE:
while True:
file_seed = self._file_seed_cache.GetNextFileSeed(status)
if file_seed is None or HG.view_shutdown:
break
path = file_seed.file_seed_data
try:
dest_dir = self._action_locations[status]
if not os.path.exists(dest_dir):
raise Exception(
'Tried to move "{}" to "{}", but the destination directory did not exist.'
.format(path, dest_dir))
if os.path.exists(path) and not os.path.isdir(path):
filename = os.path.basename(path)
dest_path = os.path.join(dest_dir, filename)
dest_path = HydrusPaths.AppendPathUntilNoConflicts(
dest_path)
HydrusPaths.MergeFile(path, dest_path)
txt_path = path + '.txt'
if os.path.exists(txt_path):
txt_filename = os.path.basename(txt_path)
txt_dest_path = os.path.join(
dest_dir, txt_filename)
txt_dest_path = HydrusPaths.AppendPathUntilNoConflicts(
txt_dest_path)
HydrusPaths.MergeFile(txt_path, txt_dest_path)
self._file_seed_cache.RemoveFileSeeds((file_seed, ))
except Exception as e:
HydrusData.ShowText('Import folder tried to move ' +
path + ', but could not:')
HydrusData.ShowException(e)
HydrusData.ShowText('Import folder has been paused.')
self._paused = True
return
elif status == CC.IMPORT_FOLDER_IGNORE:
pass
def __init__( self, media, target_resolution = None, init_position = 0 ):
RasterContainer.__init__( self, media, target_resolution )
self._init_position = init_position
self._initialised = False
self._renderer = None
self._frames = {}
self._buffer_start_index = -1
self._buffer_end_index = -1
self._times_to_play_gif = 0
self._stop = False
self._render_event = threading.Event()
( x, y ) = self._target_resolution
new_options = HG.client_controller.new_options
video_buffer_size_mb = new_options.GetInteger( 'video_buffer_size_mb' )
duration = self._media.GetDuration()
num_frames_in_video = self._media.GetNumFrames()
if duration is None or duration == 0:
message = 'The file with hash ' + media.GetHash().hex() + ', had an invalid duration.'
message += os.linesep * 2
message += 'You may wish to try regenerating its metadata through the advanced mode right-click menu.'
HydrusData.ShowText( message )
duration = 1.0
if num_frames_in_video is None or num_frames_in_video == 0:
message = 'The file with hash ' + media.GetHash().hex() + ', had an invalid number of frames.'
message += os.linesep * 2
message += 'You may wish to try regenerating its metadata through the advanced mode right-click menu.'
HydrusData.ShowText( message )
num_frames_in_video = 1
self._average_frame_duration = duration / num_frames_in_video
frame_buffer_length = ( video_buffer_size_mb * 1024 * 1024 ) // ( x * y * 3 )
# if we can't buffer the whole vid, then don't have a clunky massive buffer
max_streaming_buffer_size = max( 48, int( num_frames_in_video / ( duration / 3.0 ) ) ) # 48 or 3 seconds
if max_streaming_buffer_size < frame_buffer_length and frame_buffer_length < num_frames_in_video:
frame_buffer_length = max_streaming_buffer_size
self._num_frames_backwards = frame_buffer_length * 2 // 3
self._num_frames_forwards = frame_buffer_length // 3
self._lock = threading.Lock()
self._last_index_rendered = -1
self._next_render_index = -1
self._rendered_first_frame = False
self._ideal_next_frame = 0
HG.client_controller.CallToThread( self.THREADRender )
def MainLoop(self):
while not HydrusThreading.IsThreadShuttingDown():
time.sleep(0.00001)
with self._lock:
do_wait = len(self._waterfall_queue) == 0 and len(
self._delayed_regeneration_queue) == 0
if do_wait:
self._waterfall_event.wait(1)
self._waterfall_event.clear()
start_time = HydrusData.GetNowPrecise()
stop_time = start_time + 0.005 # a bit of a typical frame
page_keys_to_rendered_medias = collections.defaultdict(list)
num_done = 0
max_at_once = 16
while not HydrusData.TimeHasPassedPrecise(
stop_time) and num_done <= max_at_once:
with self._lock:
if len(self._waterfall_queue) == 0:
break
result = self._waterfall_queue.pop()
if len(self._waterfall_queue) == 0:
self._waterfall_queue_empty_event.set()
self._waterfall_queue_quick.discard(result)
(page_key, media) = result
if media.GetDisplayMedia() is not None:
self.GetThumbnail(media)
page_keys_to_rendered_medias[page_key].append(media)
num_done += 1
if len(page_keys_to_rendered_medias) > 0:
for (page_key,
rendered_medias) in page_keys_to_rendered_medias.items():
self._controller.pub('waterfall_thumbnails', page_key,
rendered_medias)
time.sleep(0.00001)
# now we will do regen if appropriate
with self._lock:
# got more important work or no work to do
if len(self._waterfall_queue) > 0 or len(
self._delayed_regeneration_queue
) == 0 or HG.client_controller.CurrentlyPubSubbing():
continue
media_result = self._delayed_regeneration_queue.pop()
self._delayed_regeneration_queue_quick.discard(media_result)
if HG.file_report_mode:
hash = media_result.GetHash()
HydrusData.ShowText(
'Thumbnail {} now regenerating from source.'.format(
hash.hex()))
try:
self._controller.files_maintenance_manager.RunJobImmediately(
[media_result],
ClientFiles.REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL,
pub_job_key=False)
except HydrusExceptions.FileMissingException:
pass
except Exception as e:
hash = media_result.GetHash()
summary = 'The thumbnail for file {} was incorrect, but a later attempt to regenerate it or load the new file back failed.'.format(
hash.hex())
self._HandleThumbnailException(e, summary)
def SetInitialTLWSizeAndPosition(tlw: QW.QWidget, frame_key):
new_options = HG.client_controller.new_options
(remember_size, remember_position, last_size, last_position,
default_gravity, default_position, maximised,
fullscreen) = new_options.GetFrameLocation(frame_key)
parent = tlw.parentWidget()
if parent is None:
parent_window = None
else:
parent_window = parent.window()
if remember_size and last_size is not None:
(width, height) = last_size
new_size = QC.QSize(width, height)
else:
new_size = GetSafeSize(tlw, tlw.sizeHint(), default_gravity)
tlw.resize(new_size)
min_width = min(240, new_size.width())
min_height = min(240, new_size.height())
tlw.setMinimumSize(QC.QSize(min_width, min_height))
#
child_position_point = QC.QPoint(CHILD_POSITION_PADDING,
CHILD_POSITION_PADDING)
desired_position = child_position_point
we_care_about_off_screen_messages = True
slide_up_and_left = False
if remember_position and last_position is not None:
(x, y) = last_position
desired_position = QC.QPoint(x, y)
elif default_position == 'topleft':
if parent_window is None:
we_care_about_off_screen_messages = False
screen = ClientGUIFunctions.GetMouseScreen()
if screen is not None:
desired_position = screen.availableGeometry().topLeft(
) + QC.QPoint(CHILD_POSITION_PADDING, CHILD_POSITION_PADDING)
else:
parent_tlw = parent_window.window()
desired_position = parent_tlw.pos() + QC.QPoint(
CHILD_POSITION_PADDING, CHILD_POSITION_PADDING)
slide_up_and_left = True
elif default_position == 'center':
if parent_window is None:
we_care_about_off_screen_messages = False
screen = ClientGUIFunctions.GetMouseScreen()
if screen is not None:
desired_position = screen.availableGeometry().center()
else:
desired_position = parent_window.frameGeometry().center(
) - tlw.rect().center()
(safe_position, position_message) = GetSafePosition(desired_position)
if we_care_about_off_screen_messages and position_message is not None:
HydrusData.ShowText(position_message)
if safe_position is not None:
tlw.move(safe_position)
if slide_up_and_left:
SlideOffScreenTLWUpAndLeft(tlw)
# Comment from before the Qt port: if these aren't callafter, the size and pos calls don't stick if a restore event happens
if maximised:
tlw.showMaximized()
if fullscreen and not HC.PLATFORM_MACOS:
tlw.showFullScreen()
def GetTags(self, service_key, path):
tags = set()
tags.update(self._tags_for_all)
if self._load_from_neighbouring_txt_files:
txt_path = path + '.txt'
if os.path.exists(txt_path):
try:
with open(txt_path, 'r', encoding='utf-8') as f:
txt_tags_string = f.read()
except:
HydrusData.ShowText('Could not parse the tags from ' +
txt_path + '!')
tags.add(
'___had problem reading .txt file--is it not in utf-8?'
)
try:
txt_tags = [
tag for tag in HydrusText.DeserialiseNewlinedTexts(
txt_tags_string)
]
if True in (len(txt_tag) > 1024 for txt_tag in txt_tags):
HydrusData.ShowText(
'Tags were too long--I think this was not a regular text file!'
)
raise Exception()
tags.update(txt_tags)
except:
HydrusData.ShowText('Could not parse the tags from ' +
txt_path + '!')
tags.add('___had problem parsing .txt file')
(base, filename) = os.path.split(path)
(filename, any_ext_gumpf) = os.path.splitext(filename)
(filename_boolean, filename_namespace) = self._add_filename
if filename_boolean:
if filename_namespace != '':
tag = filename_namespace + ':' + filename
else:
tag = filename
tags.add(tag)
(drive, directories) = os.path.splitdrive(base)
while directories.startswith(os.path.sep):
directories = directories[1:]
directories = directories.split(os.path.sep)
for (index, (dir_boolean,
dir_namespace)) in list(self._directories_dict.items()):
# we are talking -3 through 2 here
if not dir_boolean:
continue
try:
directory = directories[index]
except IndexError:
continue
if dir_namespace != '':
tag = dir_namespace + ':' + directory
else:
tag = directory
tags.add(tag)
#
for regex in self._regexes:
try:
result = re.findall(regex, path)
for match in result:
if isinstance(match, tuple):
for submatch in match:
tags.add(submatch)
else:
tags.add(match)
except:
pass
for (namespace, regex) in self._quick_namespaces:
try:
result = re.findall(regex, path)
for match in result:
if isinstance(match, tuple):
for submatch in match:
tags.add(namespace + ':' + submatch)
else:
tags.add(namespace + ':' + match)
except:
pass
#
tags = HydrusTags.CleanTags(tags)
tags = HG.client_controller.tag_display_manager.FilterTags(
ClientTags.TAG_DISPLAY_STORAGE, service_key, tags)
return tags
def do_it(launch_path):
if HC.PLATFORM_WINDOWS and launch_path is None:
os.startfile(path)
else:
if launch_path is None:
launch_path = GetDefaultLaunchPath()
complete_launch_path = launch_path.replace('%path%', path)
hide_terminal = False
if HC.PLATFORM_WINDOWS:
cmd = complete_launch_path
preexec_fn = None
else:
cmd = shlex.split(complete_launch_path)
preexec_fn = getattr(os, 'setsid', None)
if HG.subprocess_report_mode:
message = 'Attempting to launch ' + path + ' using command ' + repr(
cmd) + '.'
HydrusData.ShowText(message)
try:
sbp_kwargs = HydrusData.GetSubprocessKWArgs(
hide_terminal=hide_terminal, text=True)
HydrusData.CheckProgramIsNotShuttingDown()
process = subprocess.Popen(cmd,
preexec_fn=preexec_fn,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**sbp_kwargs)
(stdout,
stderr) = HydrusThreading.SubprocessCommunicate(process)
if HG.subprocess_report_mode:
if stdout is None and stderr is None:
HydrusData.ShowText('No stdout or stderr came back.')
if stdout is not None:
HydrusData.ShowText('stdout: ' + repr(stdout))
if stderr is not None:
HydrusData.ShowText('stderr: ' + repr(stderr))
except Exception as e:
HydrusData.ShowText(
'Could not launch a file! Command used was:' + os.linesep +
str(cmd))
HydrusData.ShowException(e)
def _PopulateHashIdsToHashesCache(self,
hash_ids,
exception_on_error=False):
if len(self._hash_ids_to_hashes_cache) > 100000:
if not isinstance(hash_ids, set):
hash_ids = set(hash_ids)
self._hash_ids_to_hashes_cache = {
hash_id: hash
for (hash_id, hash) in self._hash_ids_to_hashes_cache.items()
if hash_id in hash_ids
}
uncached_hash_ids = {
hash_id
for hash_id in hash_ids
if hash_id not in self._hash_ids_to_hashes_cache
}
if len(uncached_hash_ids) > 0:
pubbed_error = False
if len(uncached_hash_ids) == 1:
(uncached_hash_id, ) = uncached_hash_ids
rows = self._Execute(
'SELECT hash_id, hash FROM hashes WHERE hash_id = ?;',
(uncached_hash_id, )).fetchall()
else:
with self._MakeTemporaryIntegerTable(
uncached_hash_ids, 'hash_id') as temp_table_name:
# temp hash_ids to actual hashes
rows = self._Execute(
'SELECT hash_id, hash FROM {} CROSS JOIN hashes USING ( hash_id );'
.format(temp_table_name)).fetchall()
uncached_hash_ids_to_hashes = dict(rows)
if len(uncached_hash_ids_to_hashes) < len(uncached_hash_ids):
for hash_id in uncached_hash_ids:
if hash_id not in uncached_hash_ids_to_hashes:
if exception_on_error:
raise HydrusExceptions.DataMissing(
'Did not find all entries for those hash ids!')
HydrusData.DebugPrint('Database hash error: hash_id ' +
str(hash_id) + ' was missing!')
HydrusData.PrintException(
Exception('Missing file identifier stack trace.'))
if not pubbed_error:
HydrusData.ShowText(
'A file identifier was missing! This is a serious error that means your client database has an orphan file id! Think about contacting hydrus dev!'
)
pubbed_error = True
hash = bytes.fromhex('aaaaaaaaaaaaaaaa') + os.urandom(
16)
uncached_hash_ids_to_hashes[hash_id] = hash
self._hash_ids_to_hashes_cache.update(uncached_hash_ids_to_hashes)
def GetHashedJSONDumps(self, hashes):
shown_missing_dump_message = False
shown_broken_dump_message = False
hashes_to_objs = {}
for hash in hashes:
result = self._Execute(
'SELECT version, dump_type, dump FROM json_dumps_hashed WHERE hash = ?;',
(sqlite3.Binary(hash), )).fetchone()
if result is None:
if not shown_missing_dump_message:
message = 'A hashed serialised object was missing! Its hash is "{}".'.format(
hash.hex())
message += os.linesep * 2
message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).'
message += os.linesep * 2
message += 'Your client may have lost one or more session pages.'
message += os.linesep * 2
message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.'
HydrusData.ShowText(message)
shown_missing_dump_message = True
HydrusData.Print(
'Was asked to fetch named JSON object "{}", but it was missing!'
.format(hash.hex()))
continue
(version, dump_type, dump) = result
try:
if isinstance(dump, bytes):
dump = str(dump, 'utf-8')
serialisable_info = json.loads(dump)
except:
self._Execute('DELETE FROM json_dumps_hashed WHERE hash = ?;',
(sqlite3.Binary(hash), ))
self._cursor_transaction_wrapper.CommitAndBegin()
ExportBrokenHashedJSONDump(
self._db_dir, dump,
'hash {} dump_type {}'.format(hash.hex(), dump_type))
if not shown_broken_dump_message:
message = 'A hashed serialised object failed to load! Its hash is "{}".'.format(
hash.hex())
message += os.linesep * 2
message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).'
message += os.linesep * 2
message += 'The database has attempted to delete the broken object, and the object\'s dump written to your database directory. Your client may have lost one or more session pages.'
message += os.linesep * 2
message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.'
HydrusData.ShowText(message)
shown_broken_dump_message = True
HydrusData.Print(
'Was asked to fetch named JSON object "{}", but it was malformed!'
.format(hash.hex()))
obj = HydrusSerialisable.CreateFromSerialisableTuple(
(dump_type, version, serialisable_info))
hashes_to_objs[hash] = obj
return hashes_to_objs
def SetJSONDump(self, obj, force_timestamp=None):
if isinstance(obj, HydrusSerialisable.SerialisableBaseNamed):
(dump_type, dump_name, version,
serialisable_info) = obj.GetSerialisableTuple()
store_backups = False
backup_depth = 1
if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_GUI_SESSION_CONTAINER:
if not obj.HasAllDirtyPageData():
raise Exception(
'A session with name "{}" was set to save, but it did not have all its dirty page data!'
.format(dump_name))
hashes_to_page_data = obj.GetHashesToPageData()
self.SetHashedJSONDumps(hashes_to_page_data)
if force_timestamp is None:
store_backups = True
backup_depth = HG.client_controller.new_options.GetInteger(
'number_of_gui_session_backups')
try:
dump = json.dumps(serialisable_info)
except Exception as e:
HydrusData.ShowException(e)
HydrusData.Print(obj)
HydrusData.Print(serialisable_info)
raise Exception(
'Trying to json dump the object ' + str(obj) +
' with name ' + dump_name +
' caused an error. Its serialisable info has been dumped to the log.'
)
if force_timestamp is None:
object_timestamp = HydrusData.GetNow()
if store_backups:
existing_timestamps = sorted(
self._STI(
self._Execute(
'SELECT timestamp FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;',
(dump_type, dump_name))))
if len(existing_timestamps) > 0:
# the user has changed their system clock, so let's make sure the new timestamp is larger at least
largest_existing_timestamp = max(existing_timestamps)
if largest_existing_timestamp > object_timestamp:
object_timestamp = largest_existing_timestamp + 1
deletee_timestamps = existing_timestamps[:
-backup_depth] # keep highest n values
deletee_timestamps.append(
object_timestamp
) # if save gets spammed twice in one second, we'll overwrite
self._ExecuteMany(
'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;',
[(dump_type, dump_name, timestamp)
for timestamp in deletee_timestamps])
else:
self._Execute(
'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;',
(dump_type, dump_name))
else:
object_timestamp = force_timestamp
dump_buffer = GenerateBigSQLiteDumpBuffer(dump)
try:
self._Execute(
'INSERT INTO json_dumps_named ( dump_type, dump_name, version, timestamp, dump ) VALUES ( ?, ?, ?, ?, ? );',
(dump_type, dump_name, version, object_timestamp,
dump_buffer))
except:
HydrusData.DebugPrint(dump)
HydrusData.ShowText(
'Had a problem saving a JSON object. The dump has been printed to the log.'
)
try:
HydrusData.Print('Dump had length {}!'.format(
HydrusData.ToHumanBytes(len(dump_buffer))))
except:
pass
raise
else:
(dump_type, version,
serialisable_info) = obj.GetSerialisableTuple()
if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER:
deletee_session_names = obj.GetDeleteeSessionNames()
dirty_session_containers = obj.GetDirtySessionContainers()
if len(deletee_session_names) > 0:
for deletee_session_name in deletee_session_names:
self.DeleteJSONDumpNamed(
HydrusSerialisable.
SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER,
dump_name=deletee_session_name)
if len(dirty_session_containers) > 0:
for dirty_session_container in dirty_session_containers:
self.SetJSONDump(dirty_session_container)
if not obj.IsDirty():
return
elif dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER:
deletee_tracker_names = obj.GetDeleteeTrackerNames()
dirty_tracker_containers = obj.GetDirtyTrackerContainers()
if len(deletee_tracker_names) > 0:
for deletee_tracker_name in deletee_tracker_names:
self.DeleteJSONDumpNamed(
HydrusSerialisable.
SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER_TRACKER_CONTAINER,
dump_name=deletee_tracker_name)
if len(dirty_tracker_containers) > 0:
for dirty_tracker_container in dirty_tracker_containers:
self.SetJSONDump(dirty_tracker_container)
if not obj.IsDirty():
return
try:
dump = json.dumps(serialisable_info)
except Exception as e:
HydrusData.ShowException(e)
HydrusData.Print(obj)
HydrusData.Print(serialisable_info)
raise Exception(
'Trying to json dump the object ' + str(obj) +
' caused an error. Its serialisable info has been dumped to the log.'
)
self._Execute('DELETE FROM json_dumps WHERE dump_type = ?;',
(dump_type, ))
dump_buffer = GenerateBigSQLiteDumpBuffer(dump)
try:
self._Execute(
'INSERT INTO json_dumps ( dump_type, version, dump ) VALUES ( ?, ?, ? );',
(dump_type, version, dump_buffer))
except:
HydrusData.DebugPrint(dump)
HydrusData.ShowText(
'Had a problem saving a JSON object. The dump has been printed to the log.'
)
raise
def _DoPreCall(self):
if HG.daemon_report_mode:
HydrusData.ShowText(self._name + ' doing a job.')
def GenerateInfo(self, status_hook=None):
if self._pre_import_file_status.mime is None:
if status_hook is not None:
status_hook('generating filetype')
mime = HydrusFileHandling.GetMime(self._temp_path)
self._pre_import_file_status.mime = mime
else:
mime = self._pre_import_file_status.mime
if HG.file_import_report_mode:
HydrusData.ShowText('File import job mime: {}'.format(
HC.mime_string_lookup[mime]))
new_options = HG.client_controller.new_options
if mime in HC.DECOMPRESSION_BOMB_IMAGES and not self._file_import_options.AllowsDecompressionBombs(
):
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job testing for decompression bomb')
if HydrusImageHandling.IsDecompressionBomb(self._temp_path):
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job: it was a decompression bomb')
raise HydrusExceptions.DecompressionBombException(
'Image seems to be a Decompression Bomb!')
if status_hook is not None:
status_hook('generating file metadata')
self._file_info = HydrusFileHandling.GetFileInfo(self._temp_path,
mime=mime)
(size, mime, width, height, duration, num_frames, has_audio,
num_words) = self._file_info
if HG.file_import_report_mode:
HydrusData.ShowText('File import job file info: {}'.format(
self._file_info))
if mime in HC.MIMES_WITH_THUMBNAILS:
if status_hook is not None:
status_hook('generating thumbnail')
if HG.file_import_report_mode:
HydrusData.ShowText('File import job generating thumbnail')
bounding_dimensions = HG.client_controller.options[
'thumbnail_dimensions']
thumbnail_scale_type = HG.client_controller.new_options.GetInteger(
'thumbnail_scale_type')
(clip_rect, target_resolution
) = HydrusImageHandling.GetThumbnailResolutionAndClipRegion(
(width, height), bounding_dimensions, thumbnail_scale_type)
percentage_in = HG.client_controller.new_options.GetInteger(
'video_thumbnail_percentage_in')
try:
self._thumbnail_bytes = HydrusFileHandling.GenerateThumbnailBytes(
self._temp_path,
target_resolution,
mime,
duration,
num_frames,
clip_rect=clip_rect,
percentage_in=percentage_in)
except Exception as e:
raise HydrusExceptions.DamagedOrUnusualFileException(
'Could not render a thumbnail: {}'.format(str(e)))
if mime in HC.FILES_THAT_HAVE_PERCEPTUAL_HASH:
if status_hook is not None:
status_hook('generating similar files metadata')
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job generating perceptual_hashes')
self._perceptual_hashes = ClientImageHandling.GenerateShapePerceptualHashes(
self._temp_path, mime)
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job generated {} perceptual_hashes: {}'.
format(len(self._perceptual_hashes), [
perceptual_hash.hex()
for perceptual_hash in self._perceptual_hashes
]))
if HG.file_import_report_mode:
HydrusData.ShowText('File import job generating other hashes')
if status_hook is not None:
status_hook('generating additional hashes')
self._extra_hashes = HydrusFileHandling.GetExtraHashesFromPath(
self._temp_path)
has_icc_profile = False
if mime in HC.FILES_THAT_CAN_HAVE_ICC_PROFILE:
try:
pil_image = HydrusImageHandling.RawOpenPILImage(
self._temp_path)
has_icc_profile = HydrusImageHandling.HasICCProfile(pil_image)
except:
pass
self._has_icc_profile = has_icc_profile
if mime in HC.FILES_THAT_CAN_HAVE_PIXEL_HASH and duration is None:
try:
self._pixel_hash = HydrusImageHandling.GetImagePixelHash(
self._temp_path, mime)
except:
pass
self._file_modified_timestamp = HydrusFileHandling.GetFileModifiedTimestamp(
self._temp_path)
def DoWork(self, status_hook=None) -> FileImportStatus:
if HG.file_import_report_mode:
HydrusData.ShowText('File import job starting work.')
self.GeneratePreImportHashAndStatus(status_hook=status_hook)
if self._pre_import_file_status.ShouldImport(
self._file_import_options):
self.GenerateInfo(status_hook=status_hook)
try:
self.CheckIsGoodToImport()
ok_to_go = True
except HydrusExceptions.FileImportRulesException as e:
ok_to_go = False
not_ok_file_import_status = self._pre_import_file_status.Duplicate(
)
not_ok_file_import_status.status = CC.STATUS_VETOED
not_ok_file_import_status.note = str(e)
if ok_to_go:
hash = self._pre_import_file_status.hash
mime = self._pre_import_file_status.mime
if status_hook is not None:
status_hook('copying file into file storage')
HG.client_controller.client_files_manager.AddFile(
hash,
mime,
self._temp_path,
thumbnail_bytes=self._thumbnail_bytes)
if status_hook is not None:
status_hook('importing to database')
self._file_import_options.CheckReadyToImport()
self._post_import_file_status = HG.client_controller.WriteSynchronous(
'import_file', self)
else:
self._post_import_file_status = not_ok_file_import_status
else:
# if the file is already in the database but not in all the desired file services, let's push content updates to make it happen
if self._pre_import_file_status.status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT:
media_result = HG.client_controller.Read(
'media_result', self._pre_import_file_status.hash)
destination_location_context = self._file_import_options.GetDestinationLocationContext(
)
desired_file_service_keys = destination_location_context.current_service_keys
current_file_service_keys = media_result.GetLocationsManager(
).GetCurrent()
file_service_keys_to_add_to = set(
desired_file_service_keys).difference(
current_file_service_keys)
if len(file_service_keys_to_add_to) > 0:
file_info_manager = media_result.GetFileInfoManager()
now = HydrusData.GetNow()
service_keys_to_content_updates = {}
for service_key in file_service_keys_to_add_to:
service_keys_to_content_updates[service_key] = [
HydrusData.ContentUpdate(HC.CONTENT_TYPE_FILES,
HC.CONTENT_UPDATE_ADD,
(file_info_manager, now))
]
HG.client_controller.WriteSynchronous(
'content_updates', service_keys_to_content_updates)
self._post_import_file_status = self._pre_import_file_status.Duplicate(
)
if HG.file_import_report_mode:
HydrusData.ShowText(
'File import job is done, now publishing content updates')
self.PubsubContentUpdates()
return self._post_import_file_status
def _RefreshUPnP(self, force_wipe=False):
running_service_with_upnp = True in (service.GetPort() is not None and
service.GetUPnPPort() is not None
for service in self._services)
if not force_wipe:
if not running_service_with_upnp:
return
if running_service_with_upnp and UPNPC_IS_MISSING:
return # welp
try:
local_ip = GetLocalIP()
except:
return # can't get local IP, we are wewlad atm, probably some complicated multiple network situation we'll have to deal with later
try:
current_mappings = GetUPnPMappings()
except FileNotFoundError:
if not force_wipe:
global UPNPC_MANAGER_ERROR_PRINTED
if not UPNPC_MANAGER_ERROR_PRINTED:
HydrusData.ShowText(
'Hydrus was set up to manage your services\' port forwards with UPnP, but the miniupnpc executable is not available. Please check install_dir/bin/upnpc_readme.txt for more details.'
)
UPNPC_MANAGER_ERROR_PRINTED = True
return # in this case, most likely miniupnpc could not be found, so skip for now
except:
return # This IGD probably doesn't support UPnP, so don't spam the user with errors they can't fix!
our_mappings = {
(internal_client, internal_port): external_port
for (description, internal_client, internal_port, external_port,
protocol, enabled) in current_mappings
}
for service in self._services:
internal_port = service.GetPort()
upnp_port = service.GetUPnPPort()
if (local_ip, internal_port) in our_mappings:
current_external_port = our_mappings[(local_ip, internal_port)]
port_is_incorrect = upnp_port is None or upnp_port != current_external_port
if port_is_incorrect or force_wipe:
RemoveUPnPMapping(current_external_port, 'TCP')
for service in self._services:
internal_port = service.GetPort()
upnp_port = service.GetUPnPPort()
if upnp_port is not None:
service_type = service.GetServiceType()
protocol = 'TCP'
description = HC.service_string_lookup[
service_type] + ' at ' + local_ip + ':' + str(
internal_port)
duration = 86400
try:
AddUPnPMapping(local_ip,
internal_port,
upnp_port,
protocol,
description,
duration=duration)
except HydrusExceptions.RouterException:
HydrusData.Print(
'The UPnP Daemon tried to add {}:{}->external:{} but it failed. Please try it manually to get a full log of what happened.'
.format(local_ip, internal_port, upnp_port))
return
def VideoHasAudio(path):
info_lines = HydrusVideoHandling.GetFFMPEGInfoLines(path)
(audio_found, audio_format) = ParseFFMPEGAudio(info_lines)
if not audio_found:
return False
# just because video metadata has an audio stream doesn't mean it has audio. some vids have silent audio streams lmao
# so, let's read it as PCM and see if there is any noise
# this obviously only works for single audio stream vids, we'll adapt this if someone discovers a multi-stream mkv with a silent channel that doesn't work here
cmd = [HydrusVideoHandling.FFMPEG_PATH]
# this is perhaps not sensible for eventual playback and I should rather go for wav file-like and feed into python 'wave' in order to maintain stereo/mono and so on and have easy chunk-reading
cmd.extend(['-i', path, '-loglevel', 'quiet', '-f', 's16le', '-'])
sbp_kwargs = HydrusData.GetSubprocessKWArgs()
HydrusData.CheckProgramIsNotShuttingDown()
try:
process = subprocess.Popen(cmd,
bufsize=65536,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**sbp_kwargs)
except FileNotFoundError as e:
HydrusData.ShowText('Cannot render audio--FFMPEG not found!')
raise
# silent PCM data is just 00 bytes
# every now and then, you'll get a couple ffs for some reason, but this is not legit audio data
try:
chunk_of_pcm_data = process.stdout.read(65536)
while len(chunk_of_pcm_data) > 0:
# iterating over bytes gives you ints, recall
if True in (b != 0 and b != 255 for b in chunk_of_pcm_data):
return True
chunk_of_pcm_data = process.stdout.read(65536)
return False
finally:
process.terminate()
process.stdout.close()
process.stderr.close()
def MainLoop(self):
try:
self._InitDBCursor(
) # have to reinitialise because the thread id has changed
self._InitDiskCache()
self._InitCaches()
except:
self._DisplayCatastrophicError(traceback.format_exc())
self._could_not_initialise = True
return
self._ready_to_serve_requests = True
error_count = 0
while not ((self._local_shutdown or HG.model_shutdown)
and self._jobs.empty()):
try:
job = self._jobs.get(timeout=1)
self._currently_doing_job = True
self._current_job_name = job.ToString()
self.publish_status_update()
try:
if HG.db_report_mode:
summary = 'Running ' + job.ToString()
HydrusData.ShowText(summary)
if HG.db_profile_mode:
summary = 'Profiling ' + job.ToString()
HydrusData.ShowText(summary)
HydrusData.Profile(summary, 'self._ProcessJob( job )',
globals(), locals())
else:
self._ProcessJob(job)
error_count = 0
except:
error_count += 1
if error_count > 5:
raise
self._jobs.put(
job) # couldn't lock db; put job back on queue
time.sleep(5)
self._currently_doing_job = False
self._current_job_name = ''
self.publish_status_update()
except queue.Empty:
if self._transaction_contains_writes and HydrusData.TimeHasPassed(
self._transaction_started +
self.TRANSACTION_COMMIT_TIME):
self._Commit()
self._BeginImmediate()
self._transaction_contains_writes = False
if HydrusData.TimeHasPassed(
self._connection_timestamp + CONNECTION_REFRESH_TIME
): # just to clear out the journal files
self._InitDBCursor()
if self._pause_and_disconnect:
self._CloseDBCursor()
while self._pause_and_disconnect:
if self._local_shutdown or HG.model_shutdown:
break
time.sleep(1)
self._InitDBCursor()
self._CleanUpCaches()
self._CloseDBCursor()
temp_path = os.path.join(self._db_dir, self._durable_temp_db_filename)
HydrusPaths.DeletePath(temp_path)
self._loop_finished = True
def GenerateNumPyImage(path, mime, force_pil=False) -> numpy.array:
if HG.media_load_report_mode:
HydrusData.ShowText('Loading media: ' + path)
if not OPENCV_OK:
force_pil = True
if not force_pil:
try:
pil_image = RawOpenPILImage(path)
try:
pil_image.verify()
except:
raise HydrusExceptions.UnsupportedFileException()
# I and F are some sort of 32-bit monochrome or whatever, doesn't seem to work in PIL well, with or without ICC
if pil_image.mode not in ('I', 'F'):
if pil_image.mode == 'LAB':
force_pil = True
if HasICCProfile(pil_image):
if HG.media_load_report_mode:
HydrusData.ShowText(
'Image has ICC, so switching to PIL')
force_pil = True
except HydrusExceptions.UnsupportedFileException:
# pil had trouble, let's cross our fingers cv can do it
pass
if mime in PIL_ONLY_MIMETYPES or force_pil:
if HG.media_load_report_mode:
HydrusData.ShowText('Loading with PIL')
pil_image = GeneratePILImage(path)
numpy_image = GenerateNumPyImageFromPILImage(pil_image)
else:
if HG.media_load_report_mode:
HydrusData.ShowText('Loading with OpenCV')
if mime in (HC.IMAGE_JPEG, HC.IMAGE_TIFF):
flags = CV_IMREAD_FLAGS_JPEG
elif mime == HC.IMAGE_PNG:
flags = CV_IMREAD_FLAGS_PNG
else:
flags = CV_IMREAD_FLAGS_WEIRD
numpy_image = cv2.imread(path, flags=flags)
if numpy_image is None: # doesn't support some random stuff
if HG.media_load_report_mode:
HydrusData.ShowText('OpenCV Failed, loading with PIL')
pil_image = GeneratePILImage(path)
numpy_image = GenerateNumPyImageFromPILImage(pil_image)
else:
numpy_image = DequantizeNumPyImage(numpy_image)
if NumPyImageHasOpaqueAlphaChannel(numpy_image):
convert = cv2.COLOR_RGBA2RGB
numpy_image = cv2.cvtColor(numpy_image, convert)
return numpy_image
def OnData( self, mime_data, result ):
media_dnd = isinstance( mime_data, QMimeDataHydrusFiles )
urls_dnd = mime_data.hasUrls()
text_dnd = mime_data.hasText()
if media_dnd and self._media_callable is not None:
result = mime_data.hydrusFiles()
if result is not None:
( page_key, hashes ) = result
if page_key is not None:
QP.CallAfter( self._media_callable, page_key, hashes ) # callafter so we can terminate dnd event now
result = QC.Qt.MoveAction
# old way of doing it that messed up discord et al
'''
elif mime_data.formats().count( 'application/hydrus-media' ) and self._media_callable is not None:
mview = mime_data.data( 'application/hydrus-media' )
data_bytes = mview.data()
data_str = str( data_bytes, 'utf-8' )
(encoded_page_key, encoded_hashes) = json.loads( data_str )
if encoded_page_key is not None:
page_key = bytes.fromhex( encoded_page_key )
hashes = [ bytes.fromhex( encoded_hash ) for encoded_hash in encoded_hashes ]
QP.CallAfter( self._media_callable, page_key, hashes ) # callafter so we can terminate dnd event now
result = QC.Qt.MoveAction
'''
elif urls_dnd or text_dnd:
paths = []
urls = []
if urls_dnd:
dnd_items = mime_data.urls()
for dnd_item in dnd_items:
if dnd_item.isLocalFile():
paths.append( os.path.normpath( dnd_item.toLocalFile() ) )
else:
urls.append( dnd_item.url() )
else:
text = mime_data.text()
text_lines = HydrusText.DeserialiseNewlinedTexts( text )
for text_line in text_lines:
if text_line.startswith( 'http' ):
urls.append( text_line )
# ignore 'paths'
if self._filenames_callable is not None:
if len( paths ) > 0:
QP.CallAfter( self._filenames_callable, paths ) # callafter to terminate dnd event now
if self._url_callable is not None:
if len( urls ) > 0:
for url in urls:
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
# data:image/png;base64,(data)
# so what I prob have to do here is parse the file, decode from base64 or whatever, and then write to a fresh temp location and call self._filenames_callable
# but I need to figure out a way to reproduce this on my own. Chrome is supposed to do it on image DnD, but didn't for me
if url.startswith( 'data:' ) or len( url ) > 8 * 1024:
HydrusData.ShowText( 'This drag and drop was in the unsupported \'Data URL\' format. hydev would like to know more about this so he can fix it.' )
continue
QP.CallAfter( self._url_callable, url ) # callafter to terminate dnd event now
result = QC.Qt.IgnoreAction
else:
result = QC.Qt.IgnoreAction
return result
def _TryEndModal( self, value ):
if not self.isModal(): # in some rare cases (including spammy AutoHotkey, looks like), this can be fired before the dialog can clean itself up
return False
if not self._TestValidityAndPresentVetoMessage( value ):
return False
if not self._UserIsOKToClose( value ):
return False
if value == QW.QDialog.Rejected:
self.SetCancelled( True )
elif value == QW.QDialog.Accepted:
self._SaveOKPosition()
self._DoClose( value )
self.CleanBeforeDestroy()
try:
self.done( value )
except Exception as e:
HydrusData.ShowText( 'This dialog seems to have been unable to close for some reason. I am printing the stack to the log. The dialog may have already closed, or may attempt to close now. Please inform hydrus dev of this situation. I recommend you restart the client if you can. If the UI is locked, you will have to kill it via task manager.' )
HydrusData.PrintException( e )
import traceback
HydrusData.DebugPrint( ''.join( traceback.format_stack() ) )
try:
self.close()
except:
HydrusData.ShowText( 'The dialog would not close on command.' )
try:
self.deleteLater()
except:
HydrusData.ShowText( 'The dialog would not destroy on command.' )
return True
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
have_shown_load_error = False
( simple_key_simple_value_pairs, simple_key_serialisable_value_pairs, serialisable_key_simple_value_pairs, serialisable_key_serialisable_value_pairs ) = serialisable_info
for ( key, value ) in simple_key_simple_value_pairs:
self[ key ] = value
for ( key, serialisable_value ) in simple_key_serialisable_value_pairs:
try:
value = CreateFromSerialisableTuple( serialisable_value )
except HydrusExceptions.SerialisationException as e:
if not have_shown_load_error:
HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
HydrusData.ShowException( e )
have_shown_load_error = True
continue
self[ key ] = value
for ( serialisable_key, value ) in serialisable_key_simple_value_pairs:
try:
key = CreateFromSerialisableTuple( serialisable_key )
except HydrusExceptions.SerialisationException as e:
if not have_shown_load_error:
HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
HydrusData.ShowException( e )
have_shown_load_error = True
continue
self[ key ] = value
for ( serialisable_key, serialisable_value ) in serialisable_key_serialisable_value_pairs:
try:
key = CreateFromSerialisableTuple( serialisable_key )
value = CreateFromSerialisableTuple( serialisable_value )
except HydrusExceptions.SerialisationException as e:
if not have_shown_load_error:
HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
HydrusData.ShowException( e )
have_shown_load_error = True
continue
self[ key ] = value
def _Initialise(self):
# do this here so we are off the main thread and can wait
client_files_manager = HG.client_controller.client_files_manager
self._path = client_files_manager.GetFilePath(self._hash, self._mime)
try:
self._numpy_image = ClientImageHandling.GenerateNumPyImage(
self._path, self._mime)
except Exception as e:
HydrusData.ShowText(
'Problem rendering image at "{}"! Error follows:'.format(
self._path))
HydrusData.ShowException(e)
if not self._this_is_for_metadata_alone:
if self._numpy_image is None:
m = 'There was a problem rendering the image with hash {}! It may be damaged.'.format(
self._hash.hex())
m += os.linesep * 2
m += 'Jobs to check its integrity and metadata have been scheduled. If it is damaged, it may be redownloaded or removed from the client completely. If it is not damaged, it may be fixed automatically or further action may be required.'
HydrusData.ShowText(m)
HG.client_controller.Write(
'file_maintenance_add_jobs_hashes', {self._hash},
ClientFiles.
REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_DATA_TRY_URL_ELSE_REMOVE_RECORD
)
HG.client_controller.Write(
'file_maintenance_add_jobs_hashes', {self._hash},
ClientFiles.REGENERATE_FILE_DATA_JOB_FILE_METADATA)
else:
my_resolution_size = QC.QSize(self._resolution[0],
self._resolution[1])
my_numpy_size = QC.QSize(self._numpy_image.shape[1],
self._numpy_image.shape[0])
if my_resolution_size != my_numpy_size:
m = 'There was a problem rendering the image with hash {}! Hydrus thinks its resolution is {}, but it was actually {}.'.format(
self._hash.hex(), my_resolution_size, my_numpy_size)
m += os.linesep * 2
m += 'You may see some black squares in the image. A metadata regeneration has been scheduled, so with luck the image will fix itself soon.'
HydrusData.ShowText(m)
HG.client_controller.Write(
'file_maintenance_add_jobs_hashes', {self._hash},
ClientFiles.REGENERATE_FILE_DATA_JOB_FILE_METADATA)
def MainLoop( self ):
try:
self._InitDBCursor() # have to reinitialise because the thread id has changed
self._InitCaches()
except:
self._DisplayCatastrophicError( traceback.format_exc() )
self._could_not_initialise = True
return
self._ready_to_serve_requests = True
error_count = 0
while not ( ( self._local_shutdown or HG.model_shutdown ) and self._jobs.empty() ):
try:
job = self._jobs.get( timeout = 1 )
self._currently_doing_job = True
self._current_job_name = job.ToString()
self.publish_status_update()
try:
if HG.db_report_mode:
summary = 'Running ' + job.ToString()
HydrusData.ShowText( summary )
if HG.db_profile_mode:
summary = 'Profiling ' + job.ToString()
HydrusData.Profile( summary, 'self._ProcessJob( job )', globals(), locals(), show_summary = True )
else:
self._ProcessJob( job )
error_count = 0
except:
error_count += 1
if error_count > 5:
raise
self._jobs.put( job ) # couldn't lock db; put job back on queue
time.sleep( 5 )
self._currently_doing_job = False
self._current_job_name = ''
self.publish_status_update()
except queue.Empty:
if self._cursor_transaction_wrapper.TimeToCommit():
self._cursor_transaction_wrapper.CommitAndBegin()
if self._pause_and_disconnect:
self._CloseDBCursor()
while self._pause_and_disconnect:
if self._local_shutdown or HG.model_shutdown:
break
time.sleep( 1 )
self._InitDBCursor()
self._CloseDBCursor()
temp_path = os.path.join( self._db_dir, self._durable_temp_db_filename )
HydrusPaths.DeletePath( temp_path )
self._loop_finished = True
def GenerateNumPyImage( path, mime, force_pil = False ):
if HG.media_load_report_mode:
HydrusData.ShowText( 'Loading media: ' + path )
if not OPENCV_OK:
force_pil = True
if mime in PIL_ONLY_MIMETYPES or force_pil:
if HG.media_load_report_mode:
HydrusData.ShowText( 'Loading with PIL' )
pil_image = GeneratePILImage( path )
numpy_image = GenerateNumPyImageFromPILImage( pil_image )
else:
if HG.media_load_report_mode:
HydrusData.ShowText( 'Loading with OpenCV' )
if mime == HC.IMAGE_JPEG:
flags = CV_IMREAD_FLAGS_SUPPORTS_EXIF_REORIENTATION
else:
flags = CV_IMREAD_FLAGS_SUPPORTS_ALPHA
numpy_image = cv2.imread( path, flags = flags )
if numpy_image is None: # doesn't support static gifs and some random other stuff
if HG.media_load_report_mode:
HydrusData.ShowText( 'OpenCV Failed, loading with PIL' )
pil_image = GeneratePILImage( path )
numpy_image = GenerateNumPyImageFromPILImage( pil_image )
else:
if numpy_image.dtype == 'uint16':
numpy_image //= 256
numpy_image = numpy.array( numpy_image, dtype = 'uint8' )
shape = numpy_image.shape
if len( shape ) == 2:
# monochrome image
convert = cv2.COLOR_GRAY2RGB
else:
( im_y, im_x, depth ) = shape
if depth == 4:
convert = cv2.COLOR_BGRA2RGBA
else:
convert = cv2.COLOR_BGR2RGB
numpy_image = cv2.cvtColor( numpy_image, convert )
return numpy_image
def _GetThumbnailHydrusBitmap(self, display_media):
bounding_dimensions = self._controller.options['thumbnail_dimensions']
hash = display_media.GetHash()
mime = display_media.GetMime()
thumbnail_mime = HC.IMAGE_JPEG
# we don't actually know this, it comes down to detailed stuff, but since this is png vs jpeg it isn't a huge deal down in the guts of image loading
# ain't like I am encoding EXIF rotation in my jpeg thumbs
if mime in (HC.IMAGE_APNG, HC.IMAGE_PNG, HC.IMAGE_GIF, HC.IMAGE_ICON):
thumbnail_mime = HC.IMAGE_PNG
locations_manager = display_media.GetLocationsManager()
try:
path = self._controller.client_files_manager.GetThumbnailPath(
display_media)
except HydrusExceptions.FileMissingException as e:
if locations_manager.IsLocal():
summary = 'Unable to get thumbnail for file {}.'.format(
hash.hex())
self._HandleThumbnailException(e, summary)
return self._special_thumbs['hydrus']
try:
numpy_image = ClientImageHandling.GenerateNumPyImage(
path, thumbnail_mime)
except Exception as e:
try:
# file is malformed, let's force a regen
self._controller.files_maintenance_manager.RunJobImmediately(
[display_media],
ClientFiles.REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL,
pub_job_key=False)
except Exception as e:
summary = 'The thumbnail for file {} was not loadable. An attempt to regenerate it failed.'.format(
hash.hex())
self._HandleThumbnailException(e, summary)
return self._special_thumbs['hydrus']
try:
numpy_image = ClientImageHandling.GenerateNumPyImage(
path, thumbnail_mime)
except Exception as e:
summary = 'The thumbnail for file {} was not loadable. It was regenerated, but that file would not render either. Your image libraries or hard drive connection are unreliable. Please inform the hydrus developer what has happened.'.format(
hash.hex())
self._HandleThumbnailException(e, summary)
return self._special_thumbs['hydrus']
(current_width,
current_height) = HydrusImageHandling.GetResolutionNumPy(numpy_image)
(media_width, media_height) = display_media.GetResolution()
(expected_width,
expected_height) = HydrusImageHandling.GetThumbnailResolution(
(media_width, media_height), bounding_dimensions)
exactly_as_expected = current_width == expected_width and current_height == expected_height
rotation_exception = current_width == expected_height and current_height == expected_width
correct_size = exactly_as_expected or rotation_exception
if not correct_size:
it_is_definitely_too_big = current_width >= expected_width and current_height >= expected_height
if it_is_definitely_too_big:
if HG.file_report_mode:
HydrusData.ShowText('Thumbnail {} too big.'.format(
hash.hex()))
# the thumb we have is larger than desired. we can use it to generate what we actually want without losing significant data
# this is _resize_, not _thumbnail_, because we already know the dimensions we want
# and in some edge cases, doing getthumbresolution on existing thumb dimensions results in float/int conversion imprecision and you get 90px/91px regen cycles that never get fixed
numpy_image = HydrusImageHandling.ResizeNumPyImage(
numpy_image, (expected_width, expected_height))
if locations_manager.IsLocal():
# we have the master file, so it is safe to save our resized thumb back to disk since we can regen from source if needed
if HG.file_report_mode:
HydrusData.ShowText(
'Thumbnail {} too big, saving back to disk.'.
format(hash.hex()))
try:
try:
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesNumPy(
numpy_image, thumbnail_mime)
except HydrusExceptions.CantRenderWithCVException:
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
path, (expected_width, expected_height),
thumbnail_mime)
except:
summary = 'The thumbnail for file {} was too large, but an attempt to shrink it failed.'.format(
hash.hex())
self._HandleThumbnailException(e, summary)
return self._special_thumbs['hydrus']
try:
self._controller.client_files_manager.AddThumbnailFromBytes(
hash, thumbnail_bytes, silent=True)
self._controller.files_maintenance_manager.ClearJobs(
{hash}, ClientFiles.
REGENERATE_FILE_DATA_JOB_REFIT_THUMBNAIL)
except:
summary = 'The thumbnail for file {} was too large, but an attempt to save back the shrunk file failed.'.format(
hash.hex())
self._HandleThumbnailException(e, summary)
return self._special_thumbs['hydrus']
else:
# the thumb we have is either too small or completely messed up due to a previous ratio misparse
media_is_same_size_as_current_thumb = current_width == media_width and current_height == media_height
if media_is_same_size_as_current_thumb:
# the thumb is smaller than expected, but this is a 32x32 pixilart image or whatever, so no need to scale
if HG.file_report_mode:
HydrusData.ShowText(
'Thumbnail {} too small due to small source file.'.
format(hash.hex()))
else:
numpy_image = HydrusImageHandling.ResizeNumPyImage(
numpy_image, (expected_width, expected_height))
if locations_manager.IsLocal():
# we have the master file, so we should regen the thumb from source
if HG.file_report_mode:
HydrusData.ShowText(
'Thumbnail {} too small, scheduling regeneration from source.'
.format(hash.hex()))
delayed_item = display_media.GetMediaResult()
with self._lock:
if delayed_item not in self._delayed_regeneration_queue_quick:
self._delayed_regeneration_queue_quick.add(
delayed_item)
self._delayed_regeneration_queue.append(
delayed_item)
else:
# we do not have the master file, so we have to scale up from what we have
if HG.file_report_mode:
HydrusData.ShowText(
'Thumbnail {} was too small, only scaling up due to no local source.'
.format(hash.hex()))
hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage(
numpy_image)
return hydrus_bitmap
def GenerateNumPyImage(path, mime, force_pil=False) -> numpy.array:
if HG.media_load_report_mode:
HydrusData.ShowText('Loading media: ' + path)
if not OPENCV_OK:
force_pil = True
if not force_pil:
try:
pil_image = RawOpenPILImage(path)
if HG.media_load_report_mode:
HydrusData.ShowText('Image has ICC, so switching to PIL')
if HasICCProfile(pil_image):
force_pil = True
except HydrusExceptions.UnsupportedFileException:
# pil had trouble, let's cross our fingers cv can do it
pass
if mime in PIL_ONLY_MIMETYPES or force_pil:
if HG.media_load_report_mode:
HydrusData.ShowText('Loading with PIL')
pil_image = GeneratePILImage(path)
numpy_image = GenerateNumPyImageFromPILImage(pil_image)
else:
if HG.media_load_report_mode:
HydrusData.ShowText('Loading with OpenCV')
if mime == HC.IMAGE_JPEG:
flags = CV_IMREAD_FLAGS_SUPPORTS_EXIF_REORIENTATION
else:
flags = CV_IMREAD_FLAGS_SUPPORTS_ALPHA
numpy_image = cv2.imread(path, flags=flags)
if numpy_image is None: # doesn't support some random stuff
if HG.media_load_report_mode:
HydrusData.ShowText('OpenCV Failed, loading with PIL')
pil_image = GeneratePILImage(path)
numpy_image = GenerateNumPyImageFromPILImage(pil_image)
else:
numpy_image = DequantizeNumPyImage(numpy_image)
return numpy_image
def _WorkOnFiles(self, page_key):
file_seed = self._file_seed_cache.GetNextFileSeed(CC.STATUS_UNKNOWN)
if file_seed is None:
return
did_substantial_work = False
path = file_seed.file_seed_data
with self._lock:
self._current_action = 'importing'
def status_hook(text):
with self._lock:
if len(text) > 0:
text = text.splitlines()[0]
self._current_action = text
file_seed.ImportPath(self._file_seed_cache,
self._file_import_options,
status_hook=status_hook)
did_substantial_work = True
if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
if file_seed.ShouldPresent(self._file_import_options):
file_seed.PresentToPage(page_key)
did_substantial_work = True
if self._delete_after_success:
try:
ClientPaths.DeletePath(path)
except Exception as e:
HydrusData.ShowText('While attempting to delete ' + path +
', the following error occurred:')
HydrusData.ShowException(e)
txt_path = path + '.txt'
if os.path.exists(txt_path):
try:
ClientPaths.DeletePath(txt_path)
except Exception as e:
HydrusData.ShowText('While attempting to delete ' +
txt_path +
', the following error occurred:')
HydrusData.ShowException(e)
with self._lock:
self._current_action = ''
if did_substantial_work:
time.sleep(
ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME)
def RepopulateMissingSubtags(self, file_service_id, tag_service_id):
tags_table_name = self.GetTagsTableName(file_service_id,
tag_service_id)
subtags_fts4_table_name = self.GetSubtagsFTS4TableName(
file_service_id, tag_service_id)
subtags_searchable_map_table_name = self.GetSubtagsSearchableMapTableName(
file_service_id, tag_service_id)
integer_subtags_table_name = self.GetIntegerSubtagsTableName(
file_service_id, tag_service_id)
missing_subtag_ids = self._STS(
self._Execute(
'SELECT subtag_id FROM {} EXCEPT SELECT docid FROM {};'.format(
tags_table_name, subtags_fts4_table_name)))
for subtag_id in missing_subtag_ids:
result = self._Execute(
'SELECT subtag FROM subtags WHERE subtag_id = ?;',
(subtag_id, )).fetchone()
if result is None:
continue
(subtag, ) = result
searchable_subtag = ClientSearch.ConvertSubtagToSearchable(subtag)
if searchable_subtag != subtag:
searchable_subtag_id = self.modules_tags.GetSubtagId(
searchable_subtag)
self._Execute(
'INSERT OR IGNORE INTO {} ( subtag_id, searchable_subtag_id ) VALUES ( ?, ? );'
.format(subtags_searchable_map_table_name),
(subtag_id, searchable_subtag_id))
#
self._Execute(
'INSERT OR IGNORE INTO {} ( docid, subtag ) VALUES ( ?, ? );'.
format(subtags_fts4_table_name),
(subtag_id, searchable_subtag))
if subtag.isdecimal():
try:
integer_subtag = int(subtag)
if CanCacheInteger(integer_subtag):
self._Execute(
'INSERT OR IGNORE INTO {} ( subtag_id, integer_subtag ) VALUES ( ?, ? );'
.format(integer_subtags_table_name),
(subtag_id, integer_subtag))
except ValueError:
pass
if len(missing_subtag_ids) > 0:
HydrusData.ShowText(
'Repopulated {} missing subtags for {}_{}.'.format(
HydrusData.ToHumanInt(len(missing_subtag_ids)),
file_service_id, tag_service_id))
def _ImportFiles(self, job_key):
did_work = False
time_to_save = HydrusData.GetNow() + 600
num_files_imported = 0
presentation_hashes = []
presentation_hashes_fast = set()
i = 0
num_total = len(self._file_seed_cache)
num_total_unknown = self._file_seed_cache.GetFileSeedCount(
CC.STATUS_UNKNOWN)
num_total_done = num_total - num_total_unknown
while True:
file_seed = self._file_seed_cache.GetNextFileSeed(
CC.STATUS_UNKNOWN)
p1 = HC.options['pause_import_folders_sync'] or self._paused
p2 = HydrusThreading.IsThreadShuttingDown()
p3 = job_key.IsCancelled()
if file_seed is None or p1 or p2 or p3:
break
did_work = True
if HydrusData.TimeHasPassed(time_to_save):
HG.client_controller.WriteSynchronous('serialisable', self)
time_to_save = HydrusData.GetNow() + 600
gauge_num_done = num_total_done + num_files_imported + 1
job_key.SetVariable(
'popup_text_1',
'importing file ' + HydrusData.ConvertValueRangeToPrettyString(
gauge_num_done, num_total))
job_key.SetVariable('popup_gauge_1', (gauge_num_done, num_total))
path = file_seed.file_seed_data
file_seed.ImportPath(self._file_seed_cache,
self._file_import_options,
limited_mimes=self._mimes)
if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
if file_seed.HasHash():
hash = file_seed.GetHash()
if self._tag_import_options.HasAdditionalTags():
media_result = HG.client_controller.Read(
'media_result', hash)
downloaded_tags = []
service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates(
file_seed.status, media_result,
downloaded_tags) # additional tags
if len(service_keys_to_content_updates) > 0:
HG.client_controller.WriteSynchronous(
'content_updates',
service_keys_to_content_updates)
service_keys_to_tags = ClientTags.ServiceKeysToTags()
for (tag_service_key, filename_tagging_options) in list(
self._tag_service_keys_to_filename_tagging_options.
items()):
if not HG.client_controller.services_manager.ServiceExists(
tag_service_key):
continue
try:
tags = filename_tagging_options.GetTags(
tag_service_key, path)
if len(tags) > 0:
service_keys_to_tags[tag_service_key] = tags
except Exception as e:
HydrusData.ShowText(
'Trying to parse filename tags in the import folder "'
+ self._name + '" threw an error!')
HydrusData.ShowException(e)
if len(service_keys_to_tags) > 0:
service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates(
{hash}, service_keys_to_tags)
HG.client_controller.WriteSynchronous(
'content_updates', service_keys_to_content_updates)
num_files_imported += 1
if hash not in presentation_hashes_fast:
if file_seed.ShouldPresent(self._file_import_options):
presentation_hashes.append(hash)
presentation_hashes_fast.add(hash)
elif file_seed.status == CC.STATUS_ERROR:
HydrusData.Print(
'A file failed to import from import folder ' +
self._name + ':' + path)
i += 1
if i % 10 == 0:
self._ActionPaths()
if num_files_imported > 0:
HydrusData.Print('Import folder ' + self._name + ' imported ' +
HydrusData.ToHumanInt(num_files_imported) +
' files.')
if len(presentation_hashes) > 0:
ClientImporting.PublishPresentationHashes(
self._name, presentation_hashes,
self._publish_files_to_popup_button,
self._publish_files_to_page)
self._ActionPaths()
return did_work
def GetFFMPEGInfoLines(path,
count_frames_manually=False,
only_first_second=False):
# open the file in a pipe, provoke an error, read output
cmd = [FFMPEG_PATH, "-i", path]
if only_first_second:
cmd.insert(1, '-t')
cmd.insert(2, '1')
if count_frames_manually:
# added -an here to remove audio component, which was sometimes causing convert fails on single-frame music webms
if HC.PLATFORM_WINDOWS:
cmd += ["-vf", "scale=-2:120", "-an", "-f", "null", "NUL"]
else:
cmd += ["-vf", "scale=-2:120", "-an", "-f", "null", "/dev/null"]
sbp_kwargs = HydrusData.GetSubprocessKWArgs()
HydrusData.CheckProgramIsNotShuttingDown()
try:
process = subprocess.Popen(cmd,
bufsize=10**5,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**sbp_kwargs)
except FileNotFoundError as e:
global FFMPEG_MISSING_ERROR_PUBBED
if not FFMPEG_MISSING_ERROR_PUBBED:
message = 'FFMPEG, which hydrus uses to parse and render video, was not found! This may be due to it not being available on your system, or hydrus being unable to find it.'
message += os.linesep * 2
if HC.PLATFORM_WINDOWS:
message += 'You are on Windows, so there should be a copy of ffmpeg.exe in your install_dir/bin folder. If not, please check if your anti-virus has removed it and restore it through a new install.'
else:
message += 'If you are certain that FFMPEG is installed on your OS and accessible in your PATH, please let hydrus_dev know, as this problem is likely due to an environment problem. You may be able to solve this problem immediately by putting a static build of the ffmpeg executable in your install_dir/bin folder.'
message += os.linesep * 2
message += 'You can check your current FFMPEG status through help->about.'
HydrusData.ShowText(message)
FFMPEG_MISSING_ERROR_PUBBED = True
raise FileNotFoundError(
'Cannot interact with video because FFMPEG not found--are you sure it is installed? Full error: '
+ str(e))
(stdout, stderr) = HydrusThreading.SubprocessCommunicate(process)
data_bytes = stderr
if len(data_bytes) == 0:
global FFMPEG_NO_CONTENT_ERROR_PUBBED
if not FFMPEG_NO_CONTENT_ERROR_PUBBED:
message = 'FFMPEG, which hydrus uses to parse and render video, did not return any data on a recent file metadata check! More debug info has been written to the log.'
message += os.linesep * 2
message += 'You can check this info again through help->about.'
HydrusData.ShowText(message)
message += os.linesep * 2
message += str(sbp_kwargs)
message += os.linesep * 2
message += str(os.environ)
message += os.linesep * 2
message += 'STDOUT Response: {}'.format(stdout)
message += os.linesep * 2
message += 'STDERR Response: {}'.format(stderr)
HydrusData.DebugPrint(message)
FFMPEG_NO_CONTENT_ERROR_PUBBED = True
raise HydrusExceptions.DataMissing(
'Cannot interact with video because FFMPEG did not return any content.'
)
del process
(text, encoding) = HydrusText.NonFailingUnicodeDecode(data_bytes, 'utf-8')
lines = text.splitlines()
CheckFFMPEGError(lines)
return lines