def GenerateBigSQLiteDumpBuffer(dump): try: dump_bytes = bytes(dump, 'utf-8') except Exception as e: HydrusData.PrintException(e) raise Exception( 'While trying to save data to the database, it could not be decoded from UTF-8 to bytes! This could indicate an encoding error, such as Shift JIS sneaking into a downloader page! Please let hydrus dev know about this! Full error was written to the log!' ) if len( dump_bytes ) >= 1000000000: # 1 billion, not 1GB https://sqlite.org/limits.html raise Exception( 'A data object could not save to the database because it was bigger than a buffer limit of 1,000,000,000 bytes! If your session has a page with >500k files/URLs, reduce its size NOW or you will lose data! Otherwise, please report this to hydrus dev!' ) try: dump_buffer = sqlite3.Binary(dump_bytes) except Exception as e: HydrusData.PrintException(e) raise Exception( 'While trying to save data to the database, it would not form into a buffer! Please let hydrus dev know about this! Full error was written to the log!' ) return dump_buffer
def _TryEndModal(self, value): if not self.isModal( ): # in some rare cases (including spammy AutoHotkey, looks like), this can be fired before the dialog can clean itself up return False if not self._TestValidityAndPresentVetoMessage(value): return False if not self._UserIsOKToClose(value): return False if value == QW.QDialog.Rejected: self.SetCancelled(True) elif value == QW.QDialog.Accepted: self._SaveOKPosition() self._DoClose(value) self.CleanBeforeDestroy() try: self.done(value) except Exception as e: HydrusData.ShowText( 'This dialog seems to have been unable to close for some reason. I am printing the stack to the log. The dialog may have already closed, or may attempt to close now. Please inform hydrus dev of this situation. I recommend you restart the client if you can. If the UI is locked, you will have to kill it via task manager.' ) HydrusData.PrintException(e) import traceback HydrusData.DebugPrint(''.join(traceback.format_stack())) try: self.close() except: HydrusData.ShowText('The dialog would not close on command.') try: self.deleteLater() except: HydrusData.ShowText('The dialog would not destroy on command.') return True
def InitialiseDefaults(): global DEFAULT_HYDRUS_STYLESHEET try: with open(os.path.join(STYLESHEET_DIR, 'default_hydrus.qss'), 'r', encoding='utf-8') as f: DEFAULT_HYDRUS_STYLESHEET = f.read() except Exception as e: HydrusData.Print('Failed to load default hydrus qss:') HydrusData.PrintException(e) DEFAULT_HYDRUS_STYLESHEET = '' global ORIGINAL_STYLE_NAME global CURRENT_STYLE_NAME ORIGINAL_STYLE_NAME = QW.QApplication.instance().style().objectName() CURRENT_STYLE_NAME = ORIGINAL_STYLE_NAME global ORIGINAL_STYLESHEET global CURRENT_STYLESHEET ORIGINAL_STYLESHEET = QW.QApplication.instance().styleSheet() CURRENT_STYLESHEET = ORIGINAL_STYLESHEET
def _WorkOnFiles( self ): file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN ) if file_seed is None: return did_substantial_work = False def status_hook( text ): with self._lock: self._files_status = ClientImportControl.NeatenStatusText( text ) tag_import_options = TagImportOptions.TagImportOptions( is_default = True ) try: did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, tag_import_options ) except HydrusExceptions.NetworkException as e: delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' ) self._DelayWork( delay, str( e ) ) file_seed.SetStatus( CC.STATUS_ERROR, str( e ) ) HydrusData.PrintException( e ) except Exception as e: status = CC.STATUS_ERROR file_seed.SetStatus( status, exception = e ) time.sleep( 3 ) if file_seed.ShouldPresent( self._file_import_options.GetPresentationImportOptions() ): file_seed.PresentToPage( self._page_key ) did_substantial_work = True with self._lock: self._files_status = '' if did_substantial_work: time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
def LoadFromNumPyImage( numpy_image: numpy.array ): try: height = numpy_image.shape[0] width = numpy_image.shape[1] if len( numpy_image.shape ) > 2: depth = numpy_image.shape[2] if depth != 1: numpy_image = numpy_image[:,:,0] # let's fetch one channel. if the png is a perfect RGB conversion of the original (or, let's say, a Firefox bmp export), this actually works try: complete_data = numpy_image.tostring() top_height_header = complete_data[:2] ( top_height, ) = struct.unpack( '!H', top_height_header ) payload_and_header_bytes = complete_data[ width * top_height : ] except: raise Exception( 'Header bytes were invalid!' ) try: payload_length_header = payload_and_header_bytes[:4] ( payload_bytes_length, ) = struct.unpack( '!I', payload_length_header ) payload_bytes = payload_and_header_bytes[ 4 : 4 + payload_bytes_length ] except: raise Exception( 'Payload bytes were invalid!' ) except Exception as e: HydrusData.PrintException( e ) message = 'The image loaded, but it did not seem to be a hydrus serialised png! The error was: {}'.format( str( e ) ) message += os.linesep * 2 message += 'If you believe this is a legit non-resized, non-converted hydrus serialised png, please send it to hydrus_dev.' raise Exception( message ) return payload_bytes
def work_callable(): account_errors = set() account_keys_to_accounts = {} account_keys_to_account_info = {} for account_identifier in account_identifiers: try: result = service.Request( HC.GET, 'other_account', {'subject_identifier': account_identifier}) except Exception as e: account_errors.add(str(e)) continue if 'account' in result: account = result['account'] account_key = account.GetAccountKey() if account_key in account_keys_to_accounts: continue account_keys_to_accounts[account_key] = account try: response = self._service.Request( HC.GET, 'account_info', { 'subject_identifier': HydrusNetwork.AccountIdentifier( account_key=account_key) }) except Exception as e: HydrusData.PrintException(e) continue account_string = str(response['account_info']) account_keys_to_account_info[account_key] = account_string return (account_keys_to_accounts, account_keys_to_account_info, account_errors)
def GetQtPixmap(self, clip_rect=None, target_resolution=None): # colourspace conversions seem to be exclusively QImage territory if self._icc_profile_bytes is not None: qt_image = self.GetQtImage(clip_rect=clip_rect, target_resolution=target_resolution) return QG.QPixmap.fromImage(qt_image) (my_width, my_height) = self._resolution if clip_rect is None: clip_rect = QC.QRect(QC.QPoint(0, 0), QC.QSize(my_width, my_height)) if target_resolution is None: target_resolution = clip_rect.size() my_full_rect = QC.QRect(0, 0, my_width, my_height) if my_full_rect.contains(clip_rect): try: numpy_image = self._GetNumPyImage(clip_rect, target_resolution) (height, width, depth) = numpy_image.shape data = numpy_image.data return HG.client_controller.bitmap_manager.GetQtPixmapFromBuffer( width, height, depth * 8, data) except Exception as e: HydrusData.PrintException(e, do_wait=False) HydrusData.Print( 'Failed to produce a tile! Info is: {}, {}, {}, {}'.format( self._hash.hex(), (my_width, my_height), clip_rect, target_resolution)) pixmap = QG.QPixmap(target_resolution) pixmap.fill(QC.Qt.black) return pixmap
def _WorkOnFiles( self ): file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN ) if file_seed is None: return did_substantial_work = False url = file_seed.file_seed_data try: status_hook = lambda s: s # do nothing for now did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, self._tag_import_options ) if file_seed.ShouldPresent( self._file_import_options.GetPresentationImportOptions() ): file_seed.PresentToPage( self._page_key ) did_substantial_work = True except HydrusExceptions.NetworkException as e: delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' ) self._DelayWork( delay, str( e ) ) file_seed.SetStatus( CC.STATUS_ERROR, str( e ) ) HydrusData.PrintException( e ) except Exception as e: status = CC.STATUS_ERROR file_seed.SetStatus( status, exception = e ) time.sleep( 3 ) if did_substantial_work: time.sleep( ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )
def GetSafePosition(position: QC.QPoint, frame_key): # some window managers size the windows just off screen to cut off borders # so choose a test position that's a little more lenient fuzzy_point = QC.QPoint(FUZZY_PADDING, FUZZY_PADDING) test_position = position + fuzzy_point screen = QW.QApplication.screenAt(test_position) if screen is None: try: first_display = QW.QApplication.screens()[0] rescue_position = first_display.availableGeometry().topLeft( ) + fuzzy_point rescue_screen = QW.QApplication.screenAt(rescue_position) if rescue_screen == first_display: message = 'A window with frame key "{}" that wanted to display at "{}" was rescued from apparent off-screen to the new location at "{}".'.format( frame_key, position, rescue_position) return (rescue_position, message) except Exception as e: # user is using IceMongo Linux, a Free Libre Open Source derivation of WeasleBlue Linux, with the iJ4 5-D inverted Window Managing system, which has a holographically virtualised desktop system HydrusData.PrintException(e) message = 'A window with frame key "{}" that wanted to display at "{}" could not be rescued from off-screen! Please let hydrus dev know!'.format( frame_key, position) return (None, message) else: return (position, None)
def _WorkOnGallery( self ): gallery_seed = self._gallery_seed_log.GetNextGallerySeed( CC.STATUS_UNKNOWN ) if gallery_seed is None: return try: status_hook = lambda s: s title_hook = lambda s: s def file_seeds_callable( file_seeds ): return ClientImporting.UpdateFileSeedCacheWithFileSeeds( self._file_seed_cache, file_seeds ) gallery_seed.WorkOnURL( 'download page', self._gallery_seed_log, file_seeds_callable, status_hook, title_hook, self._NetworkJobFactory, self._GalleryNetworkJobPresentationContextFactory, self._file_import_options ) except HydrusExceptions.NetworkException as e: delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' ) self._DelayWork( delay, str( e ) ) gallery_seed.SetStatus( CC.STATUS_ERROR, str( e ) ) HydrusData.PrintException( e ) except Exception as e: status = CC.STATUS_ERROR gallery_seed.SetStatus( status, exception = e ) time.sleep( 3 ) time.sleep( 1 )
def _ProcessJob( self, job ): job_type = job.GetType() ( action, args, kwargs ) = job.GetCallableTuple() try: if job_type in ( 'read_write', 'write' ): self._current_status = 'db write locked' self._transaction_contains_writes = True else: self._current_status = 'db read locked' self.publish_status_update() if job_type in ( 'read', 'read_write' ): result = self._Read( action, *args, **kwargs ) elif job_type in ( 'write' ): result = self._Write( action, *args, **kwargs ) if self._transaction_contains_writes and HydrusData.TimeHasPassed( self._transaction_started + self.TRANSACTION_COMMIT_TIME ): self._current_status = 'db committing' self.publish_status_update() self._Commit() self._BeginImmediate() self._transaction_contains_writes = False else: self._Save() for ( topic, args, kwargs ) in self._pubsubs: self._controller.pub( topic, *args, **kwargs ) if job.IsSynchronous(): job.PutResult( result ) except Exception as e: self._ManageDBError( job, e ) try: self._Rollback() except Exception as rollback_e: HydrusData.Print( 'When the transaction failed, attempting to rollback the database failed. Please restart the client as soon as is convenient.' ) self._in_transaction = False self._CloseDBCursor() self._InitDBCursor() HydrusData.PrintException( rollback_e ) finally: self._pubsubs = [] self._current_status = '' self.publish_status_update()
def __init__( self, controller, db_dir, db_name ): if HydrusPaths.GetFreeSpace( db_dir ) < 500 * 1048576: raise Exception( 'Sorry, it looks like the db partition has less than 500MB, please free up some space.' ) self._controller = controller self._db_dir = db_dir self._db_name = db_name self._transaction_started = 0 self._in_transaction = False self._transaction_contains_writes = False self._connection_timestamp = 0 main_db_filename = db_name if not main_db_filename.endswith( '.db' ): main_db_filename += '.db' self._db_filenames = {} self._db_filenames[ 'main' ] = main_db_filename self._durable_temp_db_filename = db_name + '.temp.db' self._InitExternalDatabases() if distutils.version.LooseVersion( sqlite3.sqlite_version ) < distutils.version.LooseVersion( '3.11.0' ): self._fast_big_transaction_wal = False else: self._fast_big_transaction_wal = True self._is_first_start = False self._is_db_updated = False self._local_shutdown = False self._pause_and_disconnect = False self._loop_finished = False self._ready_to_serve_requests = False self._could_not_initialise = False self._jobs = queue.Queue() self._pubsubs = [] self._currently_doing_job = False self._current_status = '' self._current_job_name = '' self._db = None self._c = None if os.path.exists( os.path.join( self._db_dir, self._db_filenames[ 'main' ] ) ): # open and close to clean up in case last session didn't close well self._InitDB() self._CloseDBCursor() self._InitDB() self._RepairDB() ( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone() if version > HC.SOFTWARE_VERSION: self._ReportOverupdatedDB( version ) if version < ( HC.SOFTWARE_VERSION - 15 ): self._ReportUnderupdatedDB( version ) if version < HC.SOFTWARE_VERSION - 50: raise Exception( 'Your current database version of hydrus ' + str( version ) + ' is too old for this software version ' + str( HC.SOFTWARE_VERSION ) + ' to update. Please try updating with version ' + str( version + 45 ) + ' or earlier first.' ) while version < HC.SOFTWARE_VERSION: time.sleep( self.UPDATE_WAIT ) try: self._BeginImmediate() except Exception as e: raise HydrusExceptions.DBAccessException( str( e ) ) try: self._UpdateDB( version ) self._Commit() self._is_db_updated = True except: e = Exception( 'Updating the ' + self._db_name + ' db to version ' + str( version + 1 ) + ' caused this error:' + os.linesep + traceback.format_exc() ) try: self._Rollback() except Exception as rollback_e: HydrusData.Print( 'When the update failed, attempting to rollback the database failed.' ) HydrusData.PrintException( rollback_e ) raise e ( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone() self._CloseDBCursor() self._controller.CallToThreadLongRunning( self.MainLoop ) while not self._ready_to_serve_requests: time.sleep( 0.1 ) if self._could_not_initialise: raise Exception( 'Could not initialise the db! Error written to the log!' )
def GetMime(path, ok_to_look_for_hydrus_updates=False): size = os.path.getsize(path) if size == 0: raise HydrusExceptions.ZeroSizeFileException('File is of zero length!') if ok_to_look_for_hydrus_updates and size < 64 * 1024 * 1024: with open(path, 'rb') as f: update_network_bytes = f.read() try: update = HydrusSerialisable.CreateFromNetworkBytes( update_network_bytes) if isinstance(update, HydrusNetwork.ContentUpdate): return HC.APPLICATION_HYDRUS_UPDATE_CONTENT elif isinstance(update, HydrusNetwork.DefinitionsUpdate): return HC.APPLICATION_HYDRUS_UPDATE_DEFINITIONS except: pass with open(path, 'rb') as f: bit_to_check = f.read(256) for (offsets_and_headers, mime) in headers_and_mime: it_passes = False not in (bit_to_check[offset:].startswith(header) for (offset, header) in offsets_and_headers) if it_passes: if mime in (HC.UNDETERMINED_WM, HC.UNDETERMINED_MP4): return HydrusVideoHandling.GetMime(path) elif mime == HC.UNDETERMINED_PNG: if IsPNGAnimated(bit_to_check): return HC.IMAGE_APNG else: return HC.IMAGE_PNG else: return mime if HydrusText.LooksLikeHTML(bit_to_check): return HC.TEXT_HTML # it is important this goes at the end, because ffmpeg has a billion false positives! # for instance, it once thought some hydrus update files were mpegs try: mime = HydrusVideoHandling.GetMime(path) if mime != HC.APPLICATION_UNKNOWN: return mime except HydrusExceptions.UnsupportedFileException: pass except Exception as e: HydrusData.Print('FFMPEG had trouble with: ' + path) HydrusData.PrintException(e, do_wait=False) return HC.APPLICATION_UNKNOWN
def __init__(self, controller, db_dir, db_name): if HydrusPaths.GetFreeSpace(db_dir) < 500 * 1048576: raise Exception( 'Sorry, it looks like the db partition has less than 500MB, please free up some space.' ) HydrusDBBase.DBBase.__init__(self) self._controller = controller self._db_dir = db_dir self._db_name = db_name self._modules = [] HydrusDBBase.TemporaryIntegerTableNameCache() self._ssl_cert_filename = '{}.crt'.format(self._db_name) self._ssl_key_filename = '{}.key'.format(self._db_name) self._ssl_cert_path = os.path.join(self._db_dir, self._ssl_cert_filename) self._ssl_key_path = os.path.join(self._db_dir, self._ssl_key_filename) main_db_filename = db_name if not main_db_filename.endswith('.db'): main_db_filename += '.db' self._db_filenames = {} self._db_filenames['main'] = main_db_filename self._durable_temp_db_filename = db_name + '.temp.db' durable_temp_db_path = os.path.join(self._db_dir, self._durable_temp_db_filename) if os.path.exists(durable_temp_db_path): HydrusPaths.DeletePath(durable_temp_db_path) wal_lad = durable_temp_db_path + '-wal' if os.path.exists(wal_lad): HydrusPaths.DeletePath(wal_lad) shm_lad = durable_temp_db_path + '-shm' if os.path.exists(shm_lad): HydrusPaths.DeletePath(shm_lad) HydrusData.Print( 'Found and deleted the durable temporary database on boot. The last exit was probably not clean.' ) self._InitExternalDatabases() self._is_first_start = False self._is_db_updated = False self._local_shutdown = False self._pause_and_disconnect = False self._loop_finished = False self._ready_to_serve_requests = False self._could_not_initialise = False self._jobs = queue.Queue() self._currently_doing_job = False self._current_status = '' self._current_job_name = '' self._db = None self._is_connected = False self._cursor_transaction_wrapper = None if os.path.exists( os.path.join(self._db_dir, self._db_filenames['main'])): # open and close to clean up in case last session didn't close well self._InitDB() self._CloseDBConnection() self._InitDB() (version, ) = self._Execute('SELECT version FROM version;').fetchone() if version > HC.SOFTWARE_VERSION: self._ReportOverupdatedDB(version) if version < (HC.SOFTWARE_VERSION - 15): self._ReportUnderupdatedDB(version) if version < HC.SOFTWARE_VERSION - 50: raise Exception('Your current database version of hydrus ' + str(version) + ' is too old for this software version ' + str(HC.SOFTWARE_VERSION) + ' to update. Please try updating with version ' + str(version + 45) + ' or earlier first.') self._RepairDB(version) while version < HC.SOFTWARE_VERSION: time.sleep(self.UPDATE_WAIT) try: self._cursor_transaction_wrapper.BeginImmediate() except Exception as e: raise HydrusExceptions.DBAccessException(str(e)) try: self._UpdateDB(version) self._cursor_transaction_wrapper.Commit() self._is_db_updated = True except: e = Exception('Updating the ' + self._db_name + ' db to version ' + str(version + 1) + ' caused this error:' + os.linesep + traceback.format_exc()) try: self._cursor_transaction_wrapper.Rollback() except Exception as rollback_e: HydrusData.Print( 'When the update failed, attempting to rollback the database failed.' ) HydrusData.PrintException(rollback_e) raise e (version, ) = self._Execute('SELECT version FROM version;').fetchone() self._CloseDBConnection() self._controller.CallToThreadLongRunning(self.MainLoop) while not self._ready_to_serve_requests: time.sleep(0.1) if self._could_not_initialise: raise Exception( 'Could not initialise the db! Error written to the log!')
def _ProcessJob(self, job): job_type = job.GetType() (action, args, kwargs) = job.GetCallableTuple() try: if job_type in ('read_write', 'write'): self._current_status = 'db write locked' self._cursor_transaction_wrapper.NotifyWriteOccuring() else: self._current_status = 'db read locked' self.publish_status_update() if job_type in ('read', 'read_write'): result = self._Read(action, *args, **kwargs) elif job_type in ('write'): result = self._Write(action, *args, **kwargs) if self._cursor_transaction_wrapper.TimeToCommit(): self._current_status = 'db committing' self.publish_status_update() self._cursor_transaction_wrapper.CommitAndBegin() else: self._cursor_transaction_wrapper.Save() self._DoAfterJobWork() if job.IsSynchronous(): job.PutResult(result) except Exception as e: self._ManageDBError(job, e) try: self._cursor_transaction_wrapper.Rollback() except Exception as rollback_e: HydrusData.Print( 'When the transaction failed, attempting to rollback the database failed. Please restart the client as soon as is convenient.' ) self._CloseDBCursor() self._InitDBCursor() HydrusData.PrintException(rollback_e) finally: self._CleanAfterJobWork() self._current_status = '' self.publish_status_update()
def LoadFromPNG(path): # this is to deal with unicode paths, which cv2 can't handle (os_file_handle, temp_path) = HydrusTemp.GetTempPath() try: HydrusPaths.MirrorFile(path, temp_path) try: # unchanged because we want exact byte data, no conversions or other gubbins numpy_image = cv2.imread(temp_path, flags=IMREAD_UNCHANGED) if numpy_image is None: raise Exception() except Exception as e: try: # dequantize = False because we don't want to convert to RGB pil_image = HydrusImageHandling.GeneratePILImage( temp_path, dequantize=False) numpy_image = HydrusImageHandling.GenerateNumPyImageFromPILImage( pil_image) except Exception as e: HydrusData.ShowException(e) raise Exception('That did not appear to be a valid image!') finally: HydrusTemp.CleanUpTempPath(os_file_handle, temp_path) try: height = numpy_image.shape[0] width = numpy_image.shape[1] if len(numpy_image.shape) > 2: depth = numpy_image.shape[2] if depth != 1: raise Exception('The file did not appear to be monochrome!') try: complete_data = numpy_image.tostring() top_height_header = complete_data[:2] (top_height, ) = struct.unpack('!H', top_height_header) payload_and_header_bytes = complete_data[width * top_height:] except: raise Exception('Header bytes were invalid!') try: payload_length_header = payload_and_header_bytes[:4] (payload_bytes_length, ) = struct.unpack('!I', payload_length_header) payload_bytes = payload_and_header_bytes[4:4 + payload_bytes_length] except: raise Exception('Payload bytes were invalid!') except Exception as e: HydrusData.PrintException(e) message = 'The image loaded, but it did not seem to be a hydrus serialised png! The error was: {}'.format( str(e)) message += os.linesep * 2 message += 'If you believe this is a legit non-resized, non-converted hydrus serialised png, please send it to hydrus_dev.' raise Exception(message) return payload_bytes
def _CheckWatchableURL(self): def file_seeds_callable(file_seeds): return ClientImporting.UpdateFileSeedCacheWithFileSeeds( self._file_seed_cache, file_seeds) def status_hook(text): with self._lock: if len(text) > 0: text = text.splitlines()[0] self._watcher_status = text def title_hook(text): with self._lock: if len(text) > 0: text = text.splitlines()[0] self._subject = text gallery_seed = ClientImportGallerySeeds.GallerySeed( self._url, can_generate_more_pages=False) gallery_seed.SetFixedServiceKeysToTags( self._fixed_service_keys_to_tags) self._gallery_seed_log.AddGallerySeeds((gallery_seed, )) with self._lock: self._watcher_status = 'checking' try: (num_urls_added, num_urls_already_in_file_seed_cache, num_urls_total, result_404, added_new_gallery_pages, stop_reason) = gallery_seed.WorkOnURL( 'watcher', self._gallery_seed_log, file_seeds_callable, status_hook, title_hook, self._NetworkJobFactory, self._CheckerNetworkJobPresentationContextFactory, self._file_import_options) if num_urls_added > 0: ClientImporting.WakeRepeatingJob(self._files_repeating_job) if result_404: with self._lock: self._checking_paused = True self._checking_status = ClientImporting.CHECKER_STATUS_404 if gallery_seed.status == CC.STATUS_ERROR: # the [DEAD] stuff can override watcher status, so let's give a brief time for this to display the error with self._lock: self._checking_paused = True self._watcher_status = gallery_seed.note time.sleep(5) except HydrusExceptions.NetworkException as e: delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay') self._DelayWork(delay, str(e)) HydrusData.PrintException(e) watcher_status = gallery_seed.note watcher_status_should_stick = gallery_seed.status != CC.STATUS_SUCCESSFUL_AND_NEW with self._lock: if self._check_now: self._check_now = False self._watcher_status = watcher_status self._last_check_time = HydrusData.GetNow() self._UpdateFileVelocityStatus() self._UpdateNextCheckTime() self._Compact() if not watcher_status_should_stick: time.sleep(5) with self._lock: self._watcher_status = ''
from twisted.internet import reactor except: HG.twisted_is_broke = True except Exception as e: try: from hydrus.core import HydrusData HydrusData.DebugPrint( 'Critical boot error occurred! Details written to crash.log!') HydrusData.PrintException(e) except: pass error_trace = traceback.format_exc() print(error_trace) if 'db_dir' in locals() and os.path.exists(db_dir): emergency_dir = db_dir else:
def _PopulateHashIdsToHashesCache(self, hash_ids, exception_on_error=False): if len(self._hash_ids_to_hashes_cache) > 100000: if not isinstance(hash_ids, set): hash_ids = set(hash_ids) self._hash_ids_to_hashes_cache = { hash_id: hash for (hash_id, hash) in self._hash_ids_to_hashes_cache.items() if hash_id in hash_ids } uncached_hash_ids = { hash_id for hash_id in hash_ids if hash_id not in self._hash_ids_to_hashes_cache } if len(uncached_hash_ids) > 0: pubbed_error = False if len(uncached_hash_ids) == 1: (uncached_hash_id, ) = uncached_hash_ids rows = self._Execute( 'SELECT hash_id, hash FROM hashes WHERE hash_id = ?;', (uncached_hash_id, )).fetchall() else: with self._MakeTemporaryIntegerTable( uncached_hash_ids, 'hash_id') as temp_table_name: # temp hash_ids to actual hashes rows = self._Execute( 'SELECT hash_id, hash FROM {} CROSS JOIN hashes USING ( hash_id );' .format(temp_table_name)).fetchall() uncached_hash_ids_to_hashes = dict(rows) if len(uncached_hash_ids_to_hashes) < len(uncached_hash_ids): for hash_id in uncached_hash_ids: if hash_id not in uncached_hash_ids_to_hashes: if exception_on_error: raise HydrusExceptions.DataMissing( 'Did not find all entries for those hash ids!') HydrusData.DebugPrint('Database hash error: hash_id ' + str(hash_id) + ' was missing!') HydrusData.PrintException( Exception('Missing file identifier stack trace.')) if not pubbed_error: HydrusData.ShowText( 'A file identifier was missing! This is a serious error that means your client database has an orphan file id! Think about contacting hydrus dev!' ) pubbed_error = True hash = bytes.fromhex('aaaaaaaaaaaaaaaa') + os.urandom( 16) uncached_hash_ids_to_hashes[hash_id] = hash self._hash_ids_to_hashes_cache.update(uncached_hash_ids_to_hashes)
def _WorkOnGallery( self ): if len( self._pending_jobs ) > 0: with self._lock: ( url, simple_downloader_formula ) = self._pending_jobs.pop( 0 ) self._gallery_status = 'checking ' + url error_occurred = False gallery_seed_status = CC.STATUS_ERROR parser_status = 'job not completed' gallery_seed = ClientImportGallerySeeds.GallerySeed( url, can_generate_more_pages = False ) try: self._gallery_seed_log.AddGallerySeeds( ( gallery_seed, ) ) network_job = self._NetworkJobFactory( 'GET', url ) network_job.OverrideBandwidth( 30 ) HG.client_controller.network_engine.AddJob( network_job ) with self._PageNetworkJobPresentationContextFactory( network_job ): network_job.WaitUntilDone() parsing_text = network_job.GetContentText() # parsing_context = {} parsing_context[ 'url' ] = url parsing_formula = simple_downloader_formula.GetFormula() file_seeds = [] for parsed_text in parsing_formula.Parse( parsing_context, parsing_text ): try: file_url = urllib.parse.urljoin( url, parsed_text ) file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, file_url ) file_seed.SetReferralURL( url ) file_seeds.append( file_seed ) except: continue num_new = self._file_seed_cache.AddFileSeeds( file_seeds ) if num_new > 0: ClientImporting.WakeRepeatingJob( self._files_repeating_job ) parser_status = 'page checked OK with formula "' + simple_downloader_formula.GetName() + '" - ' + HydrusData.ToHumanInt( num_new ) + ' new urls' num_already_in_file_seed_cache = len( file_seeds ) - num_new if num_already_in_file_seed_cache > 0: parser_status += ' (' + HydrusData.ToHumanInt( num_already_in_file_seed_cache ) + ' already in queue)' gallery_seed_status = CC.STATUS_SUCCESSFUL_AND_NEW except HydrusExceptions.ShutdownException: gallery_seed_status = CC.STATUS_VETOED parser_status = 'program is shutting down' return except HydrusExceptions.NotFoundException: gallery_seed_status = CC.STATUS_VETOED error_occurred = True parser_status = 'page 404' except HydrusExceptions.NetworkException as e: delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' ) self._DelayWork( delay, str( e ) ) gallery_seed_status = CC.STATUS_ERROR error_occurred = True parser_status = str( e ) HydrusData.PrintException( e ) except Exception as e: gallery_seed_status = CC.STATUS_ERROR error_occurred = True parser_status = str( e ) finally: gallery_seed_note = parser_status gallery_seed.SetStatus( gallery_seed_status, note = gallery_seed_note ) self._gallery_seed_log.NotifyGallerySeedsUpdated( ( gallery_seed, ) ) with self._lock: self._gallery_status = ClientImportControl.NeatenStatusText( parser_status ) if error_occurred: time.sleep( 5 ) return True else: with self._lock: self._gallery_status = '' return False
def THREADDownloadURLs( job_key, urls, title ): job_key.SetVariable( 'popup_title', title ) job_key.SetVariable( 'popup_text_1', 'initialising' ) num_successful = 0 num_redundant = 0 num_deleted = 0 num_failed = 0 presentation_hashes = [] presentation_hashes_fast = set() file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' ) def network_job_factory( *args, **kwargs ): network_job = ClientNetworkingJobs.NetworkJob( *args, **kwargs ) network_job.OverrideBandwidth() return network_job def status_hook( text ): if len( text ) > 0: text = text.splitlines()[0] job_key.SetVariable( 'popup_text_2', text ) network_job_presentation_context_factory = GenerateMultiplePopupNetworkJobPresentationContextFactory( job_key ) for ( i, url ) in enumerate( urls ): ( i_paused, should_quit ) = job_key.WaitIfNeeded() if should_quit: break job_key.SetVariable( 'popup_text_1', HydrusData.ConvertValueRangeToPrettyString( i + 1, len( urls ) ) ) job_key.SetVariable( 'popup_gauge_1', ( i + 1, len( urls ) ) ) file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url ) try: file_seed.DownloadAndImportRawFile( url, file_import_options, network_job_factory, network_job_presentation_context_factory, status_hook ) status = file_seed.status if status in CC.SUCCESSFUL_IMPORT_STATES: if status == CC.STATUS_SUCCESSFUL_AND_NEW: num_successful += 1 elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT: num_redundant += 1 if file_seed.HasHash(): hash = file_seed.GetHash() if hash not in presentation_hashes_fast: presentation_hashes.append( hash ) presentation_hashes_fast.add( hash ) if len( presentation_hashes ) > 0: job_key.SetVariable( 'popup_files', ( presentation_hashes, 'downloads' ) ) elif status == CC.STATUS_DELETED: num_deleted += 1 except Exception as e: num_failed += 1 HydrusData.Print( url + ' failed to import!' ) HydrusData.PrintException( e ) finally: job_key.DeleteVariable( 'popup_text_2' ) job_key.DeleteVariable( 'popup_network_job' ) text_components = [] if num_successful > 0: text_components.append( HydrusData.ToHumanInt( num_successful ) + ' successful' ) if num_redundant > 0: text_components.append( HydrusData.ToHumanInt( num_redundant ) + ' already in db' ) if num_deleted > 0: text_components.append( HydrusData.ToHumanInt( num_deleted ) + ' deleted' ) if num_failed > 0: text_components.append( HydrusData.ToHumanInt( num_failed ) + ' failed (errors written to log)' ) job_key.SetVariable( 'popup_text_1', ', '.join( text_components ) ) if len( presentation_hashes ) > 0: job_key.SetVariable( 'popup_files', ( presentation_hashes, 'downloads' ) ) job_key.DeleteVariable( 'popup_gauge_1' ) job_key.Finish()
def GetMime(path, ok_to_look_for_hydrus_updates=False): size = os.path.getsize(path) if size == 0: raise HydrusExceptions.FileSizeException('File is of zero length!') with open(path, 'rb') as f: bit_to_check = f.read(256) for (offset, header, mime) in header_and_mime: offset_bit_to_check = bit_to_check[offset:] if offset_bit_to_check.startswith(header): if mime == HC.UNDETERMINED_WM: if HydrusVideoHandling.HasVideoStream(path): return HC.VIDEO_WMV # we'll catch and verify wma later elif mime == HC.UNDETERMINED_PNG: if HydrusVideoHandling.HasVideoStream(path): return HC.IMAGE_APNG else: return HC.IMAGE_PNG else: return mime try: mime = HydrusVideoHandling.GetMime(path) if mime != HC.APPLICATION_UNKNOWN: return mime except HydrusExceptions.UnsupportedFileException: pass except Exception as e: HydrusData.Print('FFMPEG had trouble with: ' + path) HydrusData.PrintException(e, do_wait=False) if HydrusText.LooksLikeHTML(bit_to_check): return HC.TEXT_HTML if ok_to_look_for_hydrus_updates: with open(path, 'rb') as f: update_network_bytes = f.read() try: update = HydrusSerialisable.CreateFromNetworkBytes( update_network_bytes) if isinstance(update, HydrusNetwork.ContentUpdate): return HC.APPLICATION_HYDRUS_UPDATE_CONTENT elif isinstance(update, HydrusNetwork.DefinitionsUpdate): return HC.APPLICATION_HYDRUS_UPDATE_DEFINITIONS except: pass return HC.APPLICATION_UNKNOWN