def CheckNetworkDownload( self, possible_mime, num_bytes, is_complete_file_size ): if is_complete_file_size: error_prefix = 'Download was apparently ' else: error_prefix = 'Download was at least ' if possible_mime is not None: if possible_mime == HC.IMAGE_GIF and self._max_gif_size is not None and num_bytes > self._max_gif_size: raise HydrusExceptions.FileSizeException( error_prefix + HydrusData.ToHumanBytes( num_bytes ) + ' but the upper limit for gifs is ' + HydrusData.ToHumanBytes( self._max_gif_size ) + '.' ) if self._max_size is not None and num_bytes > self._max_size: raise HydrusExceptions.FileSizeException( error_prefix + HydrusData.ToHumanBytes( num_bytes ) + ' but the upper limit is ' + HydrusData.ToHumanBytes( self._max_size ) + '.' ) if is_complete_file_size: if self._min_size is not None and num_bytes < self._min_size: raise HydrusExceptions.FileSizeException( error_prefix + HydrusData.ToHumanBytes( num_bytes ) + ' but the lower limit is ' + HydrusData.ToHumanBytes( self._min_size ) + '.' )
def GetSummary( self ): statements = [] if self._exclude_deleted: statements.append( 'excluding previously deleted' ) if not self._allow_decompression_bombs: statements.append( 'excluding decompression bombs' ) if self._min_size is not None: statements.append( 'excluding < ' + HydrusData.ToHumanBytes( self._min_size ) ) if self._max_size is not None: statements.append( 'excluding > ' + HydrusData.ToHumanBytes( self._max_size ) ) if self._max_gif_size is not None: statements.append( 'excluding gifs > ' + HydrusData.ToHumanBytes( self._max_gif_size ) ) if self._min_resolution is not None: ( width, height ) = self._min_resolution statements.append( 'excluding < ( ' + HydrusData.ToHumanInt( width ) + ' x ' + HydrusData.ToHumanInt( height ) + ' )' ) if self._max_resolution is not None: ( width, height ) = self._max_resolution statements.append( 'excluding > ( ' + HydrusData.ToHumanInt( width ) + ' x ' + HydrusData.ToHumanInt( height ) + ' )' ) # if self._automatic_archive: statements.append( 'automatically archiving' ) # statements.append( self._presentation_import_options.GetSummary() ) # summary = os.linesep.join( statements ) return summary
def AddData(self, key, data): with self._lock: if key not in self._keys_to_data: while self._total_estimated_memory_footprint > self._cache_size: self._DeleteItem() self._keys_to_data[key] = data self._TouchKey(key) self._RecalcMemoryUsage() if HG.cache_report_mode: HydrusData.ShowText( 'Cache "{}" adding "{}" ({}). Current size {}.'.format( self._name, key, HydrusData.ToHumanBytes( data.GetEstimatedMemoryFootprint()), HydrusData.ConvertValueRangeToBytes( self._total_estimated_memory_footprint, self._cache_size)))
def GetPayloadDescriptionAndBytes( payload_obj ): payload_bytes = GetPayloadBytes( payload_obj ) payload_description = GetPayloadTypeString( payload_obj ) + ' - ' + HydrusData.ToHumanBytes( len( payload_bytes ) ) return ( payload_description, payload_bytes )
def GetPayloadDescriptionAndBytes( payload_obj ): ( payload_bytes, payload_length ) = GetPayloadBytesAndLength( payload_obj ) payload_description = GetPayloadTypeString( payload_obj ) + ' - ' + HydrusData.ToHumanBytes( payload_length ) return ( payload_description, payload_bytes )
def ConvertBandwidthRuleToString(rule): (bandwidth_type, time_delta, max_allowed) = rule if max_allowed == 0: return 'No requests currently permitted.' if bandwidth_type == HC.BANDWIDTH_TYPE_DATA: s = HydrusData.ToHumanBytes(max_allowed) elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS: s = HydrusData.ToHumanInt(max_allowed) + ' rqs' if time_delta is None: s += ' per month' else: s += ' per ' + HydrusData.TimeDeltaToPrettyTimeDelta(time_delta) return s
def SetHashedJSONDumps(self, hashes_to_objs): for (hash, obj) in hashes_to_objs.items(): if self.HaveHashedJSONDump(hash): continue (dump_type, version, serialisable_info) = obj.GetSerialisableTuple() try: dump = json.dumps(serialisable_info) except Exception as e: HydrusData.ShowException(e) HydrusData.Print(obj) HydrusData.Print(serialisable_info) raise Exception( 'Trying to json dump the hashed object ' + str(obj) + ' caused an error. Its serialisable info has been dumped to the log.' ) maintenance_tracker = MaintenanceTracker.instance() maintenance_tracker.RegisterNewHashedSerialisable(len(dump)) dump_buffer = GenerateBigSQLiteDumpBuffer(dump) try: self._c.execute( 'INSERT INTO json_dumps_hashed ( hash, dump_type, version, dump ) VALUES ( ?, ?, ?, ? );', (sqlite3.Binary(hash), dump_type, version, dump_buffer)) except: HydrusData.DebugPrint(dump) HydrusData.ShowText( 'Had a problem saving a hashed JSON object. The dump has been printed to the log.' ) try: HydrusData.Print('Dump had length {}!'.format( HydrusData.ToHumanBytes(len(dump_buffer)))) except: pass raise
def GetCurrentMonthSummary(self): with self._lock: num_bytes = self._GetUsage(HC.BANDWIDTH_TYPE_DATA, None, True) num_requests = self._GetUsage(HC.BANDWIDTH_TYPE_REQUESTS, None, True) return 'used ' + HydrusData.ToHumanBytes( num_bytes) + ' in ' + HydrusData.ToHumanInt( num_requests) + ' requests this month'
def CheckFileIsValid( self, size, mime, width, height ): if self._min_size is not None and size < self._min_size: raise HydrusExceptions.FileSizeException( 'File was ' + HydrusData.ToHumanBytes( size ) + ' but the lower limit is ' + HydrusData.ToHumanBytes( self._min_size ) + '.' ) if self._max_size is not None and size > self._max_size: raise HydrusExceptions.FileSizeException( 'File was ' + HydrusData.ToHumanBytes( size ) + ' but the upper limit is ' + HydrusData.ToHumanBytes( self._max_size ) + '.' ) if mime == HC.IMAGE_GIF and self._max_gif_size is not None and size > self._max_gif_size: raise HydrusExceptions.FileSizeException( 'File was ' + HydrusData.ToHumanBytes( size ) + ' but the upper limit for gifs is ' + HydrusData.ToHumanBytes( self._max_gif_size ) + '.' ) if self._min_resolution is not None: ( min_width, min_height ) = self._min_resolution too_thin = width is not None and width < min_width too_short = height is not None and height < min_height if too_thin or too_short: raise HydrusExceptions.FileSizeException( 'File had resolution ' + HydrusData.ConvertResolutionToPrettyString( ( width, height ) ) + ' but the lower limit is ' + HydrusData.ConvertResolutionToPrettyString( self._min_resolution ) ) if self._max_resolution is not None: ( max_width, max_height ) = self._max_resolution too_wide = width is not None and width > max_width too_tall = height is not None and height > max_height if too_wide or too_tall: raise HydrusExceptions.FileSizeException( 'File had resolution ' + HydrusData.ConvertResolutionToPrettyString( ( width, height ) ) + ' but the upper limit is ' + HydrusData.ConvertResolutionToPrettyString( self._max_resolution ) )
def _ConvertRuleToListCtrlTuples(self, rule): (bandwidth_type, time_delta, max_allowed) = rule pretty_time_delta = HydrusData.TimeDeltaToPrettyTimeDelta(time_delta) if bandwidth_type == HC.BANDWIDTH_TYPE_DATA: pretty_max_allowed = HydrusData.ToHumanBytes(max_allowed) elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS: pretty_max_allowed = HydrusData.ToHumanInt( max_allowed) + ' requests' sort_time_delta = ClientGUIListCtrl.SafeNoneInt(time_delta) sort_tuple = (max_allowed, sort_time_delta) display_tuple = (pretty_max_allowed, pretty_time_delta) return (display_tuple, sort_tuple)
def _RenderItemName(self, name, size): return name + ' - ' + HydrusData.ToHumanBytes(size)
def _Update( self ): if self._network_job is None or self._network_job.NoEngineYet(): can_cancel = False self._left_text.clear() self._right_text.clear() self._gauge.SetRange( 1 ) self._gauge.SetValue( 0 ) else: can_cancel = not self._network_job.IsDone() ( status_text, current_speed, bytes_read, bytes_to_read ) = self._network_job.GetStatus() self._left_text.setText( status_text ) speed_text = '' if bytes_read is not None and bytes_read > 0 and not self._network_job.HasError(): if bytes_to_read is not None and bytes_read != bytes_to_read: speed_text += HydrusData.ConvertValueRangeToBytes( bytes_read, bytes_to_read ) else: speed_text += HydrusData.ToHumanBytes( bytes_read ) if current_speed != bytes_to_read: # if it is a real quick download, just say its size speed_text += ' ' + HydrusData.ToHumanBytes( current_speed ) + '/s' show_right_text = speed_text != '' if self._right_text.isVisible() != show_right_text: self._right_text.setVisible( show_right_text ) self.updateGeometry() if speed_text != '': self._right_text.setText( speed_text ) right_width = ClientGUIFunctions.ConvertTextToPixelWidth( self._right_text, len( speed_text ) ) right_min_width = right_width if right_min_width != self._last_right_min_width: self._last_right_min_width = right_min_width self._right_text.setMinimumWidth( right_min_width ) self.updateGeometry() self._gauge.SetRange( bytes_to_read ) self._gauge.SetValue( bytes_read ) if self._cancel_button.isEnabled() != can_cancel: self._cancel_button.setEnabled( can_cancel )
def __init__( self, parent, payload_obj, title = None, description = None, payload_description = None ): ClientGUIScrolledPanels.ReviewPanel.__init__( self, parent ) self._payload_obj = payload_obj self._filepicker = QP.FilePickerCtrl( self, wildcard = 'PNG (*.png)' ) self._filepicker.SetSaveMode( True ) flp_width = ClientGUIFunctions.ConvertTextToPixelWidth( self._filepicker, 64 ) self._filepicker.setMinimumWidth( flp_width ) self._title = QW.QLineEdit( self ) self._payload_description = QW.QLineEdit( self ) self._text = QW.QLineEdit( self ) self._width = QP.MakeQSpinBox( self, min=100, max=4096 ) self._export = ClientGUICommon.BetterButton( self, 'export', self.Export ) # if payload_description is None: ( payload_description, payload_bytes ) = ClientSerialisable.GetPayloadDescriptionAndBytes( self._payload_obj ) else: payload_bytes = ClientSerialisable.GetPayloadBytes( self._payload_obj ) payload_description += ' - ' + HydrusData.ToHumanBytes( len( payload_bytes ) ) self._payload_description.setText( payload_description ) self._payload_description.setEnabled( False ) self._width.setValue( 512 ) last_png_export_dir = HG.client_controller.new_options.GetNoneableString( 'last_png_export_dir' ) if title is not None: name = title elif isinstance( self._payload_obj, HydrusSerialisable.SerialisableBaseNamed ): name = self._payload_obj.GetName() else: name = payload_description self._title.setText( name ) if description is not None: self._text.setText( description ) if last_png_export_dir is not None: filename = name + '.png' filename = HydrusPaths.SanitizeFilename( filename ) path = os.path.join( last_png_export_dir, filename ) self._filepicker.SetPath( path ) self._Update() # rows = [] rows.append( ( 'export path: ', self._filepicker ) ) rows.append( ( 'title: ', self._title ) ) rows.append( ( 'payload description: ', self._payload_description ) ) rows.append( ( 'your description (optional): ', self._text ) ) rows.append( ( 'png width: ', self._width ) ) rows.append( ( '', self._export ) ) gridbox = ClientGUICommon.WrapInGrid( self, rows ) self.widget().setLayout( gridbox ) self._filepicker.filePickerChanged.connect( self._Update ) self._title.textChanged.connect( self._Update )
def _Update( self ): if self._network_job is None or self._network_job.NoEngineYet(): self._left_text.setText( '' ) self._right_text.setText( '' ) self._gauge.SetRange( 1 ) self._gauge.SetValue( 0 ) can_cancel = False else: if self._network_job.IsDone(): can_cancel = False else: can_cancel = True ( status_text, current_speed, bytes_read, bytes_to_read ) = self._network_job.GetStatus() self._left_text.setText( status_text ) if not self._download_started and current_speed > 0: self._download_started = True speed_text = '' if self._download_started and not self._network_job.HasError(): if bytes_read is not None: if bytes_to_read is not None and bytes_read != bytes_to_read: speed_text += HydrusData.ConvertValueRangeToBytes( bytes_read, bytes_to_read ) else: speed_text += HydrusData.ToHumanBytes( bytes_read ) if current_speed != bytes_to_read: # if it is a real quick download, just say its size speed_text += ' ' + HydrusData.ToHumanBytes( current_speed ) + '/s' self._right_text.setText( speed_text ) right_width = ClientGUIFunctions.ConvertTextToPixelWidth( self._right_text, len( speed_text ) ) right_min_width = right_width if right_min_width != self._last_right_min_width: self._last_right_min_width = right_min_width self._right_text.setMinimumWidth( right_min_width ) self._gauge.SetRange( bytes_to_read ) self._gauge.SetValue( bytes_read ) if can_cancel: if not self._cancel_button.isEnabled(): self._cancel_button.setEnabled( True ) else: if self._cancel_button.isEnabled(): self._cancel_button.setEnabled( False )
def CheckHasSpaceForDBTransaction(db_dir, num_bytes): if HG.no_db_temp_files: space_needed = int(num_bytes * 1.1) approx_available_memory = psutil.virtual_memory().available * 4 / 5 if approx_available_memory < num_bytes: raise Exception( 'I believe you need about ' + HydrusData.ToHumanBytes(space_needed) + ' available memory, since you are running in no_db_temp_files mode, but you only seem to have ' + HydrusData.ToHumanBytes(approx_available_memory) + '.') db_disk_free_space = GetFreeSpace(db_dir) if db_disk_free_space < space_needed: raise Exception( 'I believe you need about ' + HydrusData.ToHumanBytes(space_needed) + ' on your db\'s partition, but you only seem to have ' + HydrusData.ToHumanBytes(db_disk_free_space) + '.') else: temp_dir = tempfile.gettempdir() temp_disk_free_space = GetFreeSpace(temp_dir) temp_and_db_on_same_device = GetDevice(temp_dir) == GetDevice(db_dir) if temp_and_db_on_same_device: space_needed = int(num_bytes * 2.2) if temp_disk_free_space < space_needed: raise Exception( 'I believe you need about ' + HydrusData.ToHumanBytes(space_needed) + ' on your db\'s partition, which I think also holds your temporary path, but you only seem to have ' + HydrusData.ToHumanBytes(temp_disk_free_space) + '.') else: space_needed = int(num_bytes * 1.1) if temp_disk_free_space < space_needed: raise Exception( 'I believe you need about ' + HydrusData.ToHumanBytes(space_needed) + ' on your temporary path\'s partition, which I think is ' + temp_dir + ', but you only seem to have ' + HydrusData.ToHumanBytes(temp_disk_free_space) + '.') db_disk_free_space = GetFreeSpace(db_dir) if db_disk_free_space < space_needed: raise Exception( 'I believe you need about ' + HydrusData.ToHumanBytes(space_needed) + ' on your db\'s partition, but you only seem to have ' + HydrusData.ToHumanBytes(db_disk_free_space) + '.')
def SetJSONDump(self, obj, force_timestamp=None): if isinstance(obj, HydrusSerialisable.SerialisableBaseNamed): (dump_type, dump_name, version, serialisable_info) = obj.GetSerialisableTuple() store_backups = False backup_depth = 1 if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_GUI_SESSION_CONTAINER: if not obj.HasAllPageData(): raise Exception( 'A session with name "{}" was set to save, but it did not have all its page data!' .format(dump_name)) hashes_to_page_data = obj.GetHashesToPageData() self.SetHashedJSONDumps(hashes_to_page_data) if force_timestamp is None: store_backups = True backup_depth = HG.client_controller.new_options.GetInteger( 'number_of_gui_session_backups') try: dump = json.dumps(serialisable_info) except Exception as e: HydrusData.ShowException(e) HydrusData.Print(obj) HydrusData.Print(serialisable_info) raise Exception( 'Trying to json dump the object ' + str(obj) + ' with name ' + dump_name + ' caused an error. Its serialisable info has been dumped to the log.' ) if force_timestamp is None: object_timestamp = HydrusData.GetNow() if store_backups: existing_timestamps = sorted( self._STI( self._c.execute( 'SELECT timestamp FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;', (dump_type, dump_name)))) if len(existing_timestamps) > 0: # the user has changed their system clock, so let's make sure the new timestamp is larger at least largest_existing_timestamp = max(existing_timestamps) if largest_existing_timestamp > object_timestamp: object_timestamp = largest_existing_timestamp + 1 deletee_timestamps = existing_timestamps[: -backup_depth] # keep highest n values deletee_timestamps.append( object_timestamp ) # if save gets spammed twice in one second, we'll overwrite self._c.executemany( 'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;', [(dump_type, dump_name, timestamp) for timestamp in deletee_timestamps]) else: self._c.execute( 'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;', (dump_type, dump_name)) else: object_timestamp = force_timestamp dump_buffer = GenerateBigSQLiteDumpBuffer(dump) try: self._c.execute( 'INSERT INTO json_dumps_named ( dump_type, dump_name, version, timestamp, dump ) VALUES ( ?, ?, ?, ?, ? );', (dump_type, dump_name, version, object_timestamp, dump_buffer)) except: HydrusData.DebugPrint(dump) HydrusData.ShowText( 'Had a problem saving a JSON object. The dump has been printed to the log.' ) try: HydrusData.Print('Dump had length {}!'.format( HydrusData.ToHumanBytes(len(dump_buffer)))) except: pass raise else: (dump_type, version, serialisable_info) = obj.GetSerialisableTuple() if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER: deletee_session_names = obj.GetDeleteeSessionNames() dirty_session_containers = obj.GetDirtySessionContainers() if len(deletee_session_names) > 0: for deletee_session_name in deletee_session_names: self.DeleteJSONDumpNamed( HydrusSerialisable. SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER, dump_name=deletee_session_name) if len(dirty_session_containers) > 0: for dirty_session_container in dirty_session_containers: self.SetJSONDump(dirty_session_container) if not obj.IsDirty(): return elif dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER: deletee_tracker_names = obj.GetDeleteeTrackerNames() dirty_tracker_containers = obj.GetDirtyTrackerContainers() if len(deletee_tracker_names) > 0: for deletee_tracker_name in deletee_tracker_names: self.DeleteJSONDumpNamed( HydrusSerialisable. SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER_TRACKER_CONTAINER, dump_name=deletee_tracker_name) if len(dirty_tracker_containers) > 0: for dirty_tracker_container in dirty_tracker_containers: self.SetJSONDump(dirty_tracker_container) if not obj.IsDirty(): return try: dump = json.dumps(serialisable_info) except Exception as e: HydrusData.ShowException(e) HydrusData.Print(obj) HydrusData.Print(serialisable_info) raise Exception( 'Trying to json dump the object ' + str(obj) + ' caused an error. Its serialisable info has been dumped to the log.' ) self._c.execute('DELETE FROM json_dumps WHERE dump_type = ?;', (dump_type, )) dump_buffer = GenerateBigSQLiteDumpBuffer(dump) try: self._c.execute( 'INSERT INTO json_dumps ( dump_type, version, dump ) VALUES ( ?, ?, ? );', (dump_type, version, dump_buffer)) except: HydrusData.DebugPrint(dump) HydrusData.ShowText( 'Had a problem saving a JSON object. The dump has been printed to the log.' ) raise
def GetSummary(self): statements = [] if self._exclude_deleted: statements.append('excluding previously deleted') if not self._allow_decompression_bombs: statements.append('excluding decompression bombs') if self._min_size is not None: statements.append('excluding < ' + HydrusData.ToHumanBytes(self._min_size)) if self._max_size is not None: statements.append('excluding > ' + HydrusData.ToHumanBytes(self._max_size)) if self._max_gif_size is not None: statements.append('excluding gifs > ' + HydrusData.ToHumanBytes(self._max_gif_size)) if self._min_resolution is not None: (width, height) = self._min_resolution statements.append('excluding < ( ' + HydrusData.ToHumanInt(width) + ' x ' + HydrusData.ToHumanInt(height) + ' )') if self._max_resolution is not None: (width, height) = self._max_resolution statements.append('excluding > ( ' + HydrusData.ToHumanInt(width) + ' x ' + HydrusData.ToHumanInt(height) + ' )') # if self._automatic_archive: statements.append('automatically archiving') # presentation_statements = [] if self._present_new_files: presentation_statements.append('new') if self._present_already_in_inbox_files: presentation_statements.append('already in inbox') if self._present_already_in_archive_files: presentation_statements.append('already in archive') if len(presentation_statements) == 0: statements.append('not presenting any files') elif len(presentation_statements) == 3: statements.append('presenting all files') else: statements.append('presenting ' + ', '.join(presentation_statements) + ' files') summary = os.linesep.join(statements) return summary