def GenerateBigSQLiteDumpBuffer(dump):
try:
dump_bytes = bytes(dump, 'utf-8')
except Exception as e:
HydrusData.PrintException(e)
raise Exception(
'While trying to save data to the database, it could not be decoded from UTF-8 to bytes! This could indicate an encoding error, such as Shift JIS sneaking into a downloader page! Please let hydrus dev know about this! Full error was written to the log!'
)
if len(
dump_bytes
) >= 1000000000: # 1 billion, not 1GB https://sqlite.org/limits.html
raise Exception(
'A data object could not save to the database because it was bigger than a buffer limit of 1,000,000,000 bytes! If your session has a page with >500k files/URLs, reduce its size NOW or you will lose data! Otherwise, please report this to hydrus dev!'
)
try:
dump_buffer = sqlite3.Binary(dump_bytes)
except Exception as e:
HydrusData.PrintException(e)
raise Exception(
'While trying to save data to the database, it would not form into a buffer! Please let hydrus dev know about this! Full error was written to the log!'
)
return dump_buffer
def _TryEndModal(self, value):
if not self.isModal(
): # in some rare cases (including spammy AutoHotkey, looks like), this can be fired before the dialog can clean itself up
return False
if not self._TestValidityAndPresentVetoMessage(value):
return False
if not self._UserIsOKToClose(value):
return False
if value == QW.QDialog.Rejected:
self.SetCancelled(True)
elif value == QW.QDialog.Accepted:
self._SaveOKPosition()
self._DoClose(value)
self.CleanBeforeDestroy()
try:
self.done(value)
except Exception as e:
HydrusData.ShowText(
'This dialog seems to have been unable to close for some reason. I am printing the stack to the log. The dialog may have already closed, or may attempt to close now. Please inform hydrus dev of this situation. I recommend you restart the client if you can. If the UI is locked, you will have to kill it via task manager.'
)
HydrusData.PrintException(e)
import traceback
HydrusData.DebugPrint(''.join(traceback.format_stack()))
try:
self.close()
except:
HydrusData.ShowText('The dialog would not close on command.')
try:
self.deleteLater()
except:
HydrusData.ShowText('The dialog would not destroy on command.')
return True
def InitialiseDefaults():
global DEFAULT_HYDRUS_STYLESHEET
try:
with open(os.path.join(STYLESHEET_DIR, 'default_hydrus.qss'),
'r',
encoding='utf-8') as f:
DEFAULT_HYDRUS_STYLESHEET = f.read()
except Exception as e:
HydrusData.Print('Failed to load default hydrus qss:')
HydrusData.PrintException(e)
DEFAULT_HYDRUS_STYLESHEET = ''
global ORIGINAL_STYLE_NAME
global CURRENT_STYLE_NAME
ORIGINAL_STYLE_NAME = QW.QApplication.instance().style().objectName()
CURRENT_STYLE_NAME = ORIGINAL_STYLE_NAME
global ORIGINAL_STYLESHEET
global CURRENT_STYLESHEET
ORIGINAL_STYLESHEET = QW.QApplication.instance().styleSheet()
CURRENT_STYLESHEET = ORIGINAL_STYLESHEET
def _WorkOnFiles( self ):
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
if file_seed is None:
return
did_substantial_work = False
def status_hook( text ):
with self._lock:
self._files_status = ClientImportControl.NeatenStatusText( text )
tag_import_options = TagImportOptions.TagImportOptions( is_default = True )
try:
did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, tag_import_options )
except HydrusExceptions.NetworkException as e:
delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' )
self._DelayWork( delay, str( e ) )
file_seed.SetStatus( CC.STATUS_ERROR, str( e ) )
HydrusData.PrintException( e )
except Exception as e:
status = CC.STATUS_ERROR
file_seed.SetStatus( status, exception = e )
time.sleep( 3 )
if file_seed.ShouldPresent( self._file_import_options.GetPresentationImportOptions() ):
file_seed.PresentToPage( self._page_key )
did_substantial_work = True
with self._lock:
self._files_status = ''
if did_substantial_work:
time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
def LoadFromNumPyImage( numpy_image: numpy.array ):
try:
height = numpy_image.shape[0]
width = numpy_image.shape[1]
if len( numpy_image.shape ) > 2:
depth = numpy_image.shape[2]
if depth != 1:
numpy_image = numpy_image[:,:,0] # let's fetch one channel. if the png is a perfect RGB conversion of the original (or, let's say, a Firefox bmp export), this actually works
try:
complete_data = numpy_image.tostring()
top_height_header = complete_data[:2]
( top_height, ) = struct.unpack( '!H', top_height_header )
payload_and_header_bytes = complete_data[ width * top_height : ]
except:
raise Exception( 'Header bytes were invalid!' )
try:
payload_length_header = payload_and_header_bytes[:4]
( payload_bytes_length, ) = struct.unpack( '!I', payload_length_header )
payload_bytes = payload_and_header_bytes[ 4 : 4 + payload_bytes_length ]
except:
raise Exception( 'Payload bytes were invalid!' )
except Exception as e:
HydrusData.PrintException( e )
message = 'The image loaded, but it did not seem to be a hydrus serialised png! The error was: {}'.format( str( e ) )
message += os.linesep * 2
message += 'If you believe this is a legit non-resized, non-converted hydrus serialised png, please send it to hydrus_dev.'
raise Exception( message )
return payload_bytes
def work_callable():
account_errors = set()
account_keys_to_accounts = {}
account_keys_to_account_info = {}
for account_identifier in account_identifiers:
try:
result = service.Request(
HC.GET, 'other_account',
{'subject_identifier': account_identifier})
except Exception as e:
account_errors.add(str(e))
continue
if 'account' in result:
account = result['account']
account_key = account.GetAccountKey()
if account_key in account_keys_to_accounts:
continue
account_keys_to_accounts[account_key] = account
try:
response = self._service.Request(
HC.GET, 'account_info', {
'subject_identifier':
HydrusNetwork.AccountIdentifier(
account_key=account_key)
})
except Exception as e:
HydrusData.PrintException(e)
continue
account_string = str(response['account_info'])
account_keys_to_account_info[account_key] = account_string
return (account_keys_to_accounts, account_keys_to_account_info,
account_errors)
def GetQtPixmap(self, clip_rect=None, target_resolution=None):
# colourspace conversions seem to be exclusively QImage territory
if self._icc_profile_bytes is not None:
qt_image = self.GetQtImage(clip_rect=clip_rect,
target_resolution=target_resolution)
return QG.QPixmap.fromImage(qt_image)
(my_width, my_height) = self._resolution
if clip_rect is None:
clip_rect = QC.QRect(QC.QPoint(0, 0),
QC.QSize(my_width, my_height))
if target_resolution is None:
target_resolution = clip_rect.size()
my_full_rect = QC.QRect(0, 0, my_width, my_height)
if my_full_rect.contains(clip_rect):
try:
numpy_image = self._GetNumPyImage(clip_rect, target_resolution)
(height, width, depth) = numpy_image.shape
data = numpy_image.data
return HG.client_controller.bitmap_manager.GetQtPixmapFromBuffer(
width, height, depth * 8, data)
except Exception as e:
HydrusData.PrintException(e, do_wait=False)
HydrusData.Print(
'Failed to produce a tile! Info is: {}, {}, {}, {}'.format(
self._hash.hex(), (my_width, my_height), clip_rect,
target_resolution))
pixmap = QG.QPixmap(target_resolution)
pixmap.fill(QC.Qt.black)
return pixmap
def _WorkOnFiles( self ):
file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
if file_seed is None:
return
did_substantial_work = False
url = file_seed.file_seed_data
try:
status_hook = lambda s: s # do nothing for now
did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, self._tag_import_options )
if file_seed.ShouldPresent( self._file_import_options.GetPresentationImportOptions() ):
file_seed.PresentToPage( self._page_key )
did_substantial_work = True
except HydrusExceptions.NetworkException as e:
delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' )
self._DelayWork( delay, str( e ) )
file_seed.SetStatus( CC.STATUS_ERROR, str( e ) )
HydrusData.PrintException( e )
except Exception as e:
status = CC.STATUS_ERROR
file_seed.SetStatus( status, exception = e )
time.sleep( 3 )
if did_substantial_work:
time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
def GetSafePosition(position: QC.QPoint, frame_key):
# some window managers size the windows just off screen to cut off borders
# so choose a test position that's a little more lenient
fuzzy_point = QC.QPoint(FUZZY_PADDING, FUZZY_PADDING)
test_position = position + fuzzy_point
screen = QW.QApplication.screenAt(test_position)
if screen is None:
try:
first_display = QW.QApplication.screens()[0]
rescue_position = first_display.availableGeometry().topLeft(
) + fuzzy_point
rescue_screen = QW.QApplication.screenAt(rescue_position)
if rescue_screen == first_display:
message = 'A window with frame key "{}" that wanted to display at "{}" was rescued from apparent off-screen to the new location at "{}".'.format(
frame_key, position, rescue_position)
return (rescue_position, message)
except Exception as e:
# user is using IceMongo Linux, a Free Libre Open Source derivation of WeasleBlue Linux, with the iJ4 5-D inverted Window Managing system, which has a holographically virtualised desktop system
HydrusData.PrintException(e)
message = 'A window with frame key "{}" that wanted to display at "{}" could not be rescued from off-screen! Please let hydrus dev know!'.format(
frame_key, position)
return (None, message)
else:
return (position, None)
def _WorkOnGallery( self ):
gallery_seed = self._gallery_seed_log.GetNextGallerySeed( CC.STATUS_UNKNOWN )
if gallery_seed is None:
return
try:
status_hook = lambda s: s
title_hook = lambda s: s
def file_seeds_callable( file_seeds ):
return ClientImporting.UpdateFileSeedCacheWithFileSeeds( self._file_seed_cache, file_seeds )
gallery_seed.WorkOnURL( 'download page', self._gallery_seed_log, file_seeds_callable, status_hook, title_hook, self._NetworkJobFactory, self._GalleryNetworkJobPresentationContextFactory, self._file_import_options )
except HydrusExceptions.NetworkException as e:
delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' )
self._DelayWork( delay, str( e ) )
gallery_seed.SetStatus( CC.STATUS_ERROR, str( e ) )
HydrusData.PrintException( e )
except Exception as e:
status = CC.STATUS_ERROR
gallery_seed.SetStatus( status, exception = e )
time.sleep( 3 )
time.sleep( 1 )
def _ProcessJob( self, job ):
job_type = job.GetType()
( action, args, kwargs ) = job.GetCallableTuple()
try:
if job_type in ( 'read_write', 'write' ):
self._current_status = 'db write locked'
self._transaction_contains_writes = True
else:
self._current_status = 'db read locked'
self.publish_status_update()
if job_type in ( 'read', 'read_write' ):
result = self._Read( action, *args, **kwargs )
elif job_type in ( 'write' ):
result = self._Write( action, *args, **kwargs )
if self._transaction_contains_writes and HydrusData.TimeHasPassed( self._transaction_started + self.TRANSACTION_COMMIT_TIME ):
self._current_status = 'db committing'
self.publish_status_update()
self._Commit()
self._BeginImmediate()
self._transaction_contains_writes = False
else:
self._Save()
for ( topic, args, kwargs ) in self._pubsubs:
self._controller.pub( topic, *args, **kwargs )
if job.IsSynchronous():
job.PutResult( result )
except Exception as e:
self._ManageDBError( job, e )
try:
self._Rollback()
except Exception as rollback_e:
HydrusData.Print( 'When the transaction failed, attempting to rollback the database failed. Please restart the client as soon as is convenient.' )
self._in_transaction = False
self._CloseDBCursor()
self._InitDBCursor()
HydrusData.PrintException( rollback_e )
finally:
self._pubsubs = []
self._current_status = ''
self.publish_status_update()
def __init__( self, controller, db_dir, db_name ):
if HydrusPaths.GetFreeSpace( db_dir ) < 500 * 1048576:
raise Exception( 'Sorry, it looks like the db partition has less than 500MB, please free up some space.' )
self._controller = controller
self._db_dir = db_dir
self._db_name = db_name
self._transaction_started = 0
self._in_transaction = False
self._transaction_contains_writes = False
self._connection_timestamp = 0
main_db_filename = db_name
if not main_db_filename.endswith( '.db' ):
main_db_filename += '.db'
self._db_filenames = {}
self._db_filenames[ 'main' ] = main_db_filename
self._durable_temp_db_filename = db_name + '.temp.db'
self._InitExternalDatabases()
if distutils.version.LooseVersion( sqlite3.sqlite_version ) < distutils.version.LooseVersion( '3.11.0' ):
self._fast_big_transaction_wal = False
else:
self._fast_big_transaction_wal = True
self._is_first_start = False
self._is_db_updated = False
self._local_shutdown = False
self._pause_and_disconnect = False
self._loop_finished = False
self._ready_to_serve_requests = False
self._could_not_initialise = False
self._jobs = queue.Queue()
self._pubsubs = []
self._currently_doing_job = False
self._current_status = ''
self._current_job_name = ''
self._db = None
self._c = None
if os.path.exists( os.path.join( self._db_dir, self._db_filenames[ 'main' ] ) ):
# open and close to clean up in case last session didn't close well
self._InitDB()
self._CloseDBCursor()
self._InitDB()
self._RepairDB()
( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone()
if version > HC.SOFTWARE_VERSION:
self._ReportOverupdatedDB( version )
if version < ( HC.SOFTWARE_VERSION - 15 ):
self._ReportUnderupdatedDB( version )
if version < HC.SOFTWARE_VERSION - 50:
raise Exception( 'Your current database version of hydrus ' + str( version ) + ' is too old for this software version ' + str( HC.SOFTWARE_VERSION ) + ' to update. Please try updating with version ' + str( version + 45 ) + ' or earlier first.' )
while version < HC.SOFTWARE_VERSION:
time.sleep( self.UPDATE_WAIT )
try:
self._BeginImmediate()
except Exception as e:
raise HydrusExceptions.DBAccessException( str( e ) )
try:
self._UpdateDB( version )
self._Commit()
self._is_db_updated = True
except:
e = Exception( 'Updating the ' + self._db_name + ' db to version ' + str( version + 1 ) + ' caused this error:' + os.linesep + traceback.format_exc() )
try:
self._Rollback()
except Exception as rollback_e:
HydrusData.Print( 'When the update failed, attempting to rollback the database failed.' )
HydrusData.PrintException( rollback_e )
raise e
( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone()
self._CloseDBCursor()
self._controller.CallToThreadLongRunning( self.MainLoop )
while not self._ready_to_serve_requests:
time.sleep( 0.1 )
if self._could_not_initialise:
raise Exception( 'Could not initialise the db! Error written to the log!' )
def GetMime(path, ok_to_look_for_hydrus_updates=False):
size = os.path.getsize(path)
if size == 0:
raise HydrusExceptions.ZeroSizeFileException('File is of zero length!')
if ok_to_look_for_hydrus_updates and size < 64 * 1024 * 1024:
with open(path, 'rb') as f:
update_network_bytes = f.read()
try:
update = HydrusSerialisable.CreateFromNetworkBytes(
update_network_bytes)
if isinstance(update, HydrusNetwork.ContentUpdate):
return HC.APPLICATION_HYDRUS_UPDATE_CONTENT
elif isinstance(update, HydrusNetwork.DefinitionsUpdate):
return HC.APPLICATION_HYDRUS_UPDATE_DEFINITIONS
except:
pass
with open(path, 'rb') as f:
bit_to_check = f.read(256)
for (offsets_and_headers, mime) in headers_and_mime:
it_passes = False not in (bit_to_check[offset:].startswith(header)
for (offset, header) in offsets_and_headers)
if it_passes:
if mime in (HC.UNDETERMINED_WM, HC.UNDETERMINED_MP4):
return HydrusVideoHandling.GetMime(path)
elif mime == HC.UNDETERMINED_PNG:
if IsPNGAnimated(bit_to_check):
return HC.IMAGE_APNG
else:
return HC.IMAGE_PNG
else:
return mime
if HydrusText.LooksLikeHTML(bit_to_check):
return HC.TEXT_HTML
# it is important this goes at the end, because ffmpeg has a billion false positives!
# for instance, it once thought some hydrus update files were mpegs
try:
mime = HydrusVideoHandling.GetMime(path)
if mime != HC.APPLICATION_UNKNOWN:
return mime
except HydrusExceptions.UnsupportedFileException:
pass
except Exception as e:
HydrusData.Print('FFMPEG had trouble with: ' + path)
HydrusData.PrintException(e, do_wait=False)
return HC.APPLICATION_UNKNOWN
def __init__(self, controller, db_dir, db_name):
if HydrusPaths.GetFreeSpace(db_dir) < 500 * 1048576:
raise Exception(
'Sorry, it looks like the db partition has less than 500MB, please free up some space.'
)
HydrusDBBase.DBBase.__init__(self)
self._controller = controller
self._db_dir = db_dir
self._db_name = db_name
self._modules = []
HydrusDBBase.TemporaryIntegerTableNameCache()
self._ssl_cert_filename = '{}.crt'.format(self._db_name)
self._ssl_key_filename = '{}.key'.format(self._db_name)
self._ssl_cert_path = os.path.join(self._db_dir,
self._ssl_cert_filename)
self._ssl_key_path = os.path.join(self._db_dir, self._ssl_key_filename)
main_db_filename = db_name
if not main_db_filename.endswith('.db'):
main_db_filename += '.db'
self._db_filenames = {}
self._db_filenames['main'] = main_db_filename
self._durable_temp_db_filename = db_name + '.temp.db'
durable_temp_db_path = os.path.join(self._db_dir,
self._durable_temp_db_filename)
if os.path.exists(durable_temp_db_path):
HydrusPaths.DeletePath(durable_temp_db_path)
wal_lad = durable_temp_db_path + '-wal'
if os.path.exists(wal_lad):
HydrusPaths.DeletePath(wal_lad)
shm_lad = durable_temp_db_path + '-shm'
if os.path.exists(shm_lad):
HydrusPaths.DeletePath(shm_lad)
HydrusData.Print(
'Found and deleted the durable temporary database on boot. The last exit was probably not clean.'
)
self._InitExternalDatabases()
self._is_first_start = False
self._is_db_updated = False
self._local_shutdown = False
self._pause_and_disconnect = False
self._loop_finished = False
self._ready_to_serve_requests = False
self._could_not_initialise = False
self._jobs = queue.Queue()
self._currently_doing_job = False
self._current_status = ''
self._current_job_name = ''
self._db = None
self._is_connected = False
self._cursor_transaction_wrapper = None
if os.path.exists(
os.path.join(self._db_dir, self._db_filenames['main'])):
# open and close to clean up in case last session didn't close well
self._InitDB()
self._CloseDBConnection()
self._InitDB()
(version, ) = self._Execute('SELECT version FROM version;').fetchone()
if version > HC.SOFTWARE_VERSION:
self._ReportOverupdatedDB(version)
if version < (HC.SOFTWARE_VERSION - 15):
self._ReportUnderupdatedDB(version)
if version < HC.SOFTWARE_VERSION - 50:
raise Exception('Your current database version of hydrus ' +
str(version) +
' is too old for this software version ' +
str(HC.SOFTWARE_VERSION) +
' to update. Please try updating with version ' +
str(version + 45) + ' or earlier first.')
self._RepairDB(version)
while version < HC.SOFTWARE_VERSION:
time.sleep(self.UPDATE_WAIT)
try:
self._cursor_transaction_wrapper.BeginImmediate()
except Exception as e:
raise HydrusExceptions.DBAccessException(str(e))
try:
self._UpdateDB(version)
self._cursor_transaction_wrapper.Commit()
self._is_db_updated = True
except:
e = Exception('Updating the ' + self._db_name +
' db to version ' + str(version + 1) +
' caused this error:' + os.linesep +
traceback.format_exc())
try:
self._cursor_transaction_wrapper.Rollback()
except Exception as rollback_e:
HydrusData.Print(
'When the update failed, attempting to rollback the database failed.'
)
HydrusData.PrintException(rollback_e)
raise e
(version,
) = self._Execute('SELECT version FROM version;').fetchone()
self._CloseDBConnection()
self._controller.CallToThreadLongRunning(self.MainLoop)
while not self._ready_to_serve_requests:
time.sleep(0.1)
if self._could_not_initialise:
raise Exception(
'Could not initialise the db! Error written to the log!')
def _ProcessJob(self, job):
job_type = job.GetType()
(action, args, kwargs) = job.GetCallableTuple()
try:
if job_type in ('read_write', 'write'):
self._current_status = 'db write locked'
self._cursor_transaction_wrapper.NotifyWriteOccuring()
else:
self._current_status = 'db read locked'
self.publish_status_update()
if job_type in ('read', 'read_write'):
result = self._Read(action, *args, **kwargs)
elif job_type in ('write'):
result = self._Write(action, *args, **kwargs)
if self._cursor_transaction_wrapper.TimeToCommit():
self._current_status = 'db committing'
self.publish_status_update()
self._cursor_transaction_wrapper.CommitAndBegin()
else:
self._cursor_transaction_wrapper.Save()
self._DoAfterJobWork()
if job.IsSynchronous():
job.PutResult(result)
except Exception as e:
self._ManageDBError(job, e)
try:
self._cursor_transaction_wrapper.Rollback()
except Exception as rollback_e:
HydrusData.Print(
'When the transaction failed, attempting to rollback the database failed. Please restart the client as soon as is convenient.'
)
self._CloseDBCursor()
self._InitDBCursor()
HydrusData.PrintException(rollback_e)
finally:
self._CleanAfterJobWork()
self._current_status = ''
self.publish_status_update()
def LoadFromPNG(path):
# this is to deal with unicode paths, which cv2 can't handle
(os_file_handle, temp_path) = HydrusTemp.GetTempPath()
try:
HydrusPaths.MirrorFile(path, temp_path)
try:
# unchanged because we want exact byte data, no conversions or other gubbins
numpy_image = cv2.imread(temp_path, flags=IMREAD_UNCHANGED)
if numpy_image is None:
raise Exception()
except Exception as e:
try:
# dequantize = False because we don't want to convert to RGB
pil_image = HydrusImageHandling.GeneratePILImage(
temp_path, dequantize=False)
numpy_image = HydrusImageHandling.GenerateNumPyImageFromPILImage(
pil_image)
except Exception as e:
HydrusData.ShowException(e)
raise Exception('That did not appear to be a valid image!')
finally:
HydrusTemp.CleanUpTempPath(os_file_handle, temp_path)
try:
height = numpy_image.shape[0]
width = numpy_image.shape[1]
if len(numpy_image.shape) > 2:
depth = numpy_image.shape[2]
if depth != 1:
raise Exception('The file did not appear to be monochrome!')
try:
complete_data = numpy_image.tostring()
top_height_header = complete_data[:2]
(top_height, ) = struct.unpack('!H', top_height_header)
payload_and_header_bytes = complete_data[width * top_height:]
except:
raise Exception('Header bytes were invalid!')
try:
payload_length_header = payload_and_header_bytes[:4]
(payload_bytes_length, ) = struct.unpack('!I',
payload_length_header)
payload_bytes = payload_and_header_bytes[4:4 +
payload_bytes_length]
except:
raise Exception('Payload bytes were invalid!')
except Exception as e:
HydrusData.PrintException(e)
message = 'The image loaded, but it did not seem to be a hydrus serialised png! The error was: {}'.format(
str(e))
message += os.linesep * 2
message += 'If you believe this is a legit non-resized, non-converted hydrus serialised png, please send it to hydrus_dev.'
raise Exception(message)
return payload_bytes
def _CheckWatchableURL(self):
def file_seeds_callable(file_seeds):
return ClientImporting.UpdateFileSeedCacheWithFileSeeds(
self._file_seed_cache, file_seeds)
def status_hook(text):
with self._lock:
if len(text) > 0:
text = text.splitlines()[0]
self._watcher_status = text
def title_hook(text):
with self._lock:
if len(text) > 0:
text = text.splitlines()[0]
self._subject = text
gallery_seed = ClientImportGallerySeeds.GallerySeed(
self._url, can_generate_more_pages=False)
gallery_seed.SetFixedServiceKeysToTags(
self._fixed_service_keys_to_tags)
self._gallery_seed_log.AddGallerySeeds((gallery_seed, ))
with self._lock:
self._watcher_status = 'checking'
try:
(num_urls_added, num_urls_already_in_file_seed_cache,
num_urls_total, result_404, added_new_gallery_pages,
stop_reason) = gallery_seed.WorkOnURL(
'watcher', self._gallery_seed_log, file_seeds_callable,
status_hook, title_hook, self._NetworkJobFactory,
self._CheckerNetworkJobPresentationContextFactory,
self._file_import_options)
if num_urls_added > 0:
ClientImporting.WakeRepeatingJob(self._files_repeating_job)
if result_404:
with self._lock:
self._checking_paused = True
self._checking_status = ClientImporting.CHECKER_STATUS_404
if gallery_seed.status == CC.STATUS_ERROR:
# the [DEAD] stuff can override watcher status, so let's give a brief time for this to display the error
with self._lock:
self._checking_paused = True
self._watcher_status = gallery_seed.note
time.sleep(5)
except HydrusExceptions.NetworkException as e:
delay = HG.client_controller.new_options.GetInteger(
'downloader_network_error_delay')
self._DelayWork(delay, str(e))
HydrusData.PrintException(e)
watcher_status = gallery_seed.note
watcher_status_should_stick = gallery_seed.status != CC.STATUS_SUCCESSFUL_AND_NEW
with self._lock:
if self._check_now:
self._check_now = False
self._watcher_status = watcher_status
self._last_check_time = HydrusData.GetNow()
self._UpdateFileVelocityStatus()
self._UpdateNextCheckTime()
self._Compact()
if not watcher_status_should_stick:
time.sleep(5)
with self._lock:
self._watcher_status = ''
from twisted.internet import reactor
except:
HG.twisted_is_broke = True
except Exception as e:
try:
from hydrus.core import HydrusData
HydrusData.DebugPrint(
'Critical boot error occurred! Details written to crash.log!')
HydrusData.PrintException(e)
except:
pass
error_trace = traceback.format_exc()
print(error_trace)
if 'db_dir' in locals() and os.path.exists(db_dir):
emergency_dir = db_dir
else:
def _PopulateHashIdsToHashesCache(self,
hash_ids,
exception_on_error=False):
if len(self._hash_ids_to_hashes_cache) > 100000:
if not isinstance(hash_ids, set):
hash_ids = set(hash_ids)
self._hash_ids_to_hashes_cache = {
hash_id: hash
for (hash_id, hash) in self._hash_ids_to_hashes_cache.items()
if hash_id in hash_ids
}
uncached_hash_ids = {
hash_id
for hash_id in hash_ids
if hash_id not in self._hash_ids_to_hashes_cache
}
if len(uncached_hash_ids) > 0:
pubbed_error = False
if len(uncached_hash_ids) == 1:
(uncached_hash_id, ) = uncached_hash_ids
rows = self._Execute(
'SELECT hash_id, hash FROM hashes WHERE hash_id = ?;',
(uncached_hash_id, )).fetchall()
else:
with self._MakeTemporaryIntegerTable(
uncached_hash_ids, 'hash_id') as temp_table_name:
# temp hash_ids to actual hashes
rows = self._Execute(
'SELECT hash_id, hash FROM {} CROSS JOIN hashes USING ( hash_id );'
.format(temp_table_name)).fetchall()
uncached_hash_ids_to_hashes = dict(rows)
if len(uncached_hash_ids_to_hashes) < len(uncached_hash_ids):
for hash_id in uncached_hash_ids:
if hash_id not in uncached_hash_ids_to_hashes:
if exception_on_error:
raise HydrusExceptions.DataMissing(
'Did not find all entries for those hash ids!')
HydrusData.DebugPrint('Database hash error: hash_id ' +
str(hash_id) + ' was missing!')
HydrusData.PrintException(
Exception('Missing file identifier stack trace.'))
if not pubbed_error:
HydrusData.ShowText(
'A file identifier was missing! This is a serious error that means your client database has an orphan file id! Think about contacting hydrus dev!'
)
pubbed_error = True
hash = bytes.fromhex('aaaaaaaaaaaaaaaa') + os.urandom(
16)
uncached_hash_ids_to_hashes[hash_id] = hash
self._hash_ids_to_hashes_cache.update(uncached_hash_ids_to_hashes)
def _WorkOnGallery( self ):
if len( self._pending_jobs ) > 0:
with self._lock:
( url, simple_downloader_formula ) = self._pending_jobs.pop( 0 )
self._gallery_status = 'checking ' + url
error_occurred = False
gallery_seed_status = CC.STATUS_ERROR
parser_status = 'job not completed'
gallery_seed = ClientImportGallerySeeds.GallerySeed( url, can_generate_more_pages = False )
try:
self._gallery_seed_log.AddGallerySeeds( ( gallery_seed, ) )
network_job = self._NetworkJobFactory( 'GET', url )
network_job.OverrideBandwidth( 30 )
HG.client_controller.network_engine.AddJob( network_job )
with self._PageNetworkJobPresentationContextFactory( network_job ):
network_job.WaitUntilDone()
parsing_text = network_job.GetContentText()
#
parsing_context = {}
parsing_context[ 'url' ] = url
parsing_formula = simple_downloader_formula.GetFormula()
file_seeds = []
for parsed_text in parsing_formula.Parse( parsing_context, parsing_text ):
try:
file_url = urllib.parse.urljoin( url, parsed_text )
file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, file_url )
file_seed.SetReferralURL( url )
file_seeds.append( file_seed )
except:
continue
num_new = self._file_seed_cache.AddFileSeeds( file_seeds )
if num_new > 0:
ClientImporting.WakeRepeatingJob( self._files_repeating_job )
parser_status = 'page checked OK with formula "' + simple_downloader_formula.GetName() + '" - ' + HydrusData.ToHumanInt( num_new ) + ' new urls'
num_already_in_file_seed_cache = len( file_seeds ) - num_new
if num_already_in_file_seed_cache > 0:
parser_status += ' (' + HydrusData.ToHumanInt( num_already_in_file_seed_cache ) + ' already in queue)'
gallery_seed_status = CC.STATUS_SUCCESSFUL_AND_NEW
except HydrusExceptions.ShutdownException:
gallery_seed_status = CC.STATUS_VETOED
parser_status = 'program is shutting down'
return
except HydrusExceptions.NotFoundException:
gallery_seed_status = CC.STATUS_VETOED
error_occurred = True
parser_status = 'page 404'
except HydrusExceptions.NetworkException as e:
delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' )
self._DelayWork( delay, str( e ) )
gallery_seed_status = CC.STATUS_ERROR
error_occurred = True
parser_status = str( e )
HydrusData.PrintException( e )
except Exception as e:
gallery_seed_status = CC.STATUS_ERROR
error_occurred = True
parser_status = str( e )
finally:
gallery_seed_note = parser_status
gallery_seed.SetStatus( gallery_seed_status, note = gallery_seed_note )
self._gallery_seed_log.NotifyGallerySeedsUpdated( ( gallery_seed, ) )
with self._lock:
self._gallery_status = ClientImportControl.NeatenStatusText( parser_status )
if error_occurred:
time.sleep( 5 )
return True
else:
with self._lock:
self._gallery_status = ''
return False
def THREADDownloadURLs( job_key, urls, title ):
job_key.SetVariable( 'popup_title', title )
job_key.SetVariable( 'popup_text_1', 'initialising' )
num_successful = 0
num_redundant = 0
num_deleted = 0
num_failed = 0
presentation_hashes = []
presentation_hashes_fast = set()
file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )
def network_job_factory( *args, **kwargs ):
network_job = ClientNetworkingJobs.NetworkJob( *args, **kwargs )
network_job.OverrideBandwidth()
return network_job
def status_hook( text ):
if len( text ) > 0:
text = text.splitlines()[0]
job_key.SetVariable( 'popup_text_2', text )
network_job_presentation_context_factory = GenerateMultiplePopupNetworkJobPresentationContextFactory( job_key )
for ( i, url ) in enumerate( urls ):
( i_paused, should_quit ) = job_key.WaitIfNeeded()
if should_quit:
break
job_key.SetVariable( 'popup_text_1', HydrusData.ConvertValueRangeToPrettyString( i + 1, len( urls ) ) )
job_key.SetVariable( 'popup_gauge_1', ( i + 1, len( urls ) ) )
file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url )
try:
file_seed.DownloadAndImportRawFile( url, file_import_options, network_job_factory, network_job_presentation_context_factory, status_hook )
status = file_seed.status
if status in CC.SUCCESSFUL_IMPORT_STATES:
if status == CC.STATUS_SUCCESSFUL_AND_NEW:
num_successful += 1
elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT:
num_redundant += 1
if file_seed.HasHash():
hash = file_seed.GetHash()
if hash not in presentation_hashes_fast:
presentation_hashes.append( hash )
presentation_hashes_fast.add( hash )
if len( presentation_hashes ) > 0:
job_key.SetVariable( 'popup_files', ( presentation_hashes, 'downloads' ) )
elif status == CC.STATUS_DELETED:
num_deleted += 1
except Exception as e:
num_failed += 1
HydrusData.Print( url + ' failed to import!' )
HydrusData.PrintException( e )
finally:
job_key.DeleteVariable( 'popup_text_2' )
job_key.DeleteVariable( 'popup_network_job' )
text_components = []
if num_successful > 0:
text_components.append( HydrusData.ToHumanInt( num_successful ) + ' successful' )
if num_redundant > 0:
text_components.append( HydrusData.ToHumanInt( num_redundant ) + ' already in db' )
if num_deleted > 0:
text_components.append( HydrusData.ToHumanInt( num_deleted ) + ' deleted' )
if num_failed > 0:
text_components.append( HydrusData.ToHumanInt( num_failed ) + ' failed (errors written to log)' )
job_key.SetVariable( 'popup_text_1', ', '.join( text_components ) )
if len( presentation_hashes ) > 0:
job_key.SetVariable( 'popup_files', ( presentation_hashes, 'downloads' ) )
job_key.DeleteVariable( 'popup_gauge_1' )
job_key.Finish()
def GetMime(path, ok_to_look_for_hydrus_updates=False):
size = os.path.getsize(path)
if size == 0:
raise HydrusExceptions.FileSizeException('File is of zero length!')
with open(path, 'rb') as f:
bit_to_check = f.read(256)
for (offset, header, mime) in header_and_mime:
offset_bit_to_check = bit_to_check[offset:]
if offset_bit_to_check.startswith(header):
if mime == HC.UNDETERMINED_WM:
if HydrusVideoHandling.HasVideoStream(path):
return HC.VIDEO_WMV
# we'll catch and verify wma later
elif mime == HC.UNDETERMINED_PNG:
if HydrusVideoHandling.HasVideoStream(path):
return HC.IMAGE_APNG
else:
return HC.IMAGE_PNG
else:
return mime
try:
mime = HydrusVideoHandling.GetMime(path)
if mime != HC.APPLICATION_UNKNOWN:
return mime
except HydrusExceptions.UnsupportedFileException:
pass
except Exception as e:
HydrusData.Print('FFMPEG had trouble with: ' + path)
HydrusData.PrintException(e, do_wait=False)
if HydrusText.LooksLikeHTML(bit_to_check):
return HC.TEXT_HTML
if ok_to_look_for_hydrus_updates:
with open(path, 'rb') as f:
update_network_bytes = f.read()
try:
update = HydrusSerialisable.CreateFromNetworkBytes(
update_network_bytes)
if isinstance(update, HydrusNetwork.ContentUpdate):
return HC.APPLICATION_HYDRUS_UPDATE_CONTENT
elif isinstance(update, HydrusNetwork.DefinitionsUpdate):
return HC.APPLICATION_HYDRUS_UPDATE_DEFINITIONS
except:
pass
return HC.APPLICATION_UNKNOWN
