def CheckNetworkDownload( self, possible_mime, num_bytes, is_complete_file_size ):
if is_complete_file_size:
error_prefix = 'Download was apparently '
else:
error_prefix = 'Download was at least '
if possible_mime is not None:
if possible_mime == HC.IMAGE_GIF and self._max_gif_size is not None and num_bytes > self._max_gif_size:
raise HydrusExceptions.FileSizeException( error_prefix + HydrusData.ToHumanBytes( num_bytes ) + ' but the upper limit for gifs is ' + HydrusData.ToHumanBytes( self._max_gif_size ) + '.' )
if self._max_size is not None and num_bytes > self._max_size:
raise HydrusExceptions.FileSizeException( error_prefix + HydrusData.ToHumanBytes( num_bytes ) + ' but the upper limit is ' + HydrusData.ToHumanBytes( self._max_size ) + '.' )
if is_complete_file_size:
if self._min_size is not None and num_bytes < self._min_size:
raise HydrusExceptions.FileSizeException( error_prefix + HydrusData.ToHumanBytes( num_bytes ) + ' but the lower limit is ' + HydrusData.ToHumanBytes( self._min_size ) + '.' )
def GetSummary( self ):
statements = []
if self._exclude_deleted:
statements.append( 'excluding previously deleted' )
if not self._allow_decompression_bombs:
statements.append( 'excluding decompression bombs' )
if self._min_size is not None:
statements.append( 'excluding < ' + HydrusData.ToHumanBytes( self._min_size ) )
if self._max_size is not None:
statements.append( 'excluding > ' + HydrusData.ToHumanBytes( self._max_size ) )
if self._max_gif_size is not None:
statements.append( 'excluding gifs > ' + HydrusData.ToHumanBytes( self._max_gif_size ) )
if self._min_resolution is not None:
( width, height ) = self._min_resolution
statements.append( 'excluding < ( ' + HydrusData.ToHumanInt( width ) + ' x ' + HydrusData.ToHumanInt( height ) + ' )' )
if self._max_resolution is not None:
( width, height ) = self._max_resolution
statements.append( 'excluding > ( ' + HydrusData.ToHumanInt( width ) + ' x ' + HydrusData.ToHumanInt( height ) + ' )' )
#
if self._automatic_archive:
statements.append( 'automatically archiving' )
#
statements.append( self._presentation_import_options.GetSummary() )
#
summary = os.linesep.join( statements )
return summary
def AddData(self, key, data):
with self._lock:
if key not in self._keys_to_data:
while self._total_estimated_memory_footprint > self._cache_size:
self._DeleteItem()
self._keys_to_data[key] = data
self._TouchKey(key)
self._RecalcMemoryUsage()
if HG.cache_report_mode:
HydrusData.ShowText(
'Cache "{}" adding "{}" ({}). Current size {}.'.format(
self._name, key,
HydrusData.ToHumanBytes(
data.GetEstimatedMemoryFootprint()),
HydrusData.ConvertValueRangeToBytes(
self._total_estimated_memory_footprint,
self._cache_size)))
def GetPayloadDescriptionAndBytes( payload_obj ):
payload_bytes = GetPayloadBytes( payload_obj )
payload_description = GetPayloadTypeString( payload_obj ) + ' - ' + HydrusData.ToHumanBytes( len( payload_bytes ) )
return ( payload_description, payload_bytes )
def GetPayloadDescriptionAndBytes( payload_obj ):
( payload_bytes, payload_length ) = GetPayloadBytesAndLength( payload_obj )
payload_description = GetPayloadTypeString( payload_obj ) + ' - ' + HydrusData.ToHumanBytes( payload_length )
return ( payload_description, payload_bytes )
def ConvertBandwidthRuleToString(rule):
(bandwidth_type, time_delta, max_allowed) = rule
if max_allowed == 0:
return 'No requests currently permitted.'
if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
s = HydrusData.ToHumanBytes(max_allowed)
elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
s = HydrusData.ToHumanInt(max_allowed) + ' rqs'
if time_delta is None:
s += ' per month'
else:
s += ' per ' + HydrusData.TimeDeltaToPrettyTimeDelta(time_delta)
return s
def SetHashedJSONDumps(self, hashes_to_objs):
for (hash, obj) in hashes_to_objs.items():
if self.HaveHashedJSONDump(hash):
continue
(dump_type, version,
serialisable_info) = obj.GetSerialisableTuple()
try:
dump = json.dumps(serialisable_info)
except Exception as e:
HydrusData.ShowException(e)
HydrusData.Print(obj)
HydrusData.Print(serialisable_info)
raise Exception(
'Trying to json dump the hashed object ' + str(obj) +
' caused an error. Its serialisable info has been dumped to the log.'
)
maintenance_tracker = MaintenanceTracker.instance()
maintenance_tracker.RegisterNewHashedSerialisable(len(dump))
dump_buffer = GenerateBigSQLiteDumpBuffer(dump)
try:
self._c.execute(
'INSERT INTO json_dumps_hashed ( hash, dump_type, version, dump ) VALUES ( ?, ?, ?, ? );',
(sqlite3.Binary(hash), dump_type, version, dump_buffer))
except:
HydrusData.DebugPrint(dump)
HydrusData.ShowText(
'Had a problem saving a hashed JSON object. The dump has been printed to the log.'
)
try:
HydrusData.Print('Dump had length {}!'.format(
HydrusData.ToHumanBytes(len(dump_buffer))))
except:
pass
raise
def GetCurrentMonthSummary(self):
with self._lock:
num_bytes = self._GetUsage(HC.BANDWIDTH_TYPE_DATA, None, True)
num_requests = self._GetUsage(HC.BANDWIDTH_TYPE_REQUESTS, None,
True)
return 'used ' + HydrusData.ToHumanBytes(
num_bytes) + ' in ' + HydrusData.ToHumanInt(
num_requests) + ' requests this month'
def CheckFileIsValid( self, size, mime, width, height ):
if self._min_size is not None and size < self._min_size:
raise HydrusExceptions.FileSizeException( 'File was ' + HydrusData.ToHumanBytes( size ) + ' but the lower limit is ' + HydrusData.ToHumanBytes( self._min_size ) + '.' )
if self._max_size is not None and size > self._max_size:
raise HydrusExceptions.FileSizeException( 'File was ' + HydrusData.ToHumanBytes( size ) + ' but the upper limit is ' + HydrusData.ToHumanBytes( self._max_size ) + '.' )
if mime == HC.IMAGE_GIF and self._max_gif_size is not None and size > self._max_gif_size:
raise HydrusExceptions.FileSizeException( 'File was ' + HydrusData.ToHumanBytes( size ) + ' but the upper limit for gifs is ' + HydrusData.ToHumanBytes( self._max_gif_size ) + '.' )
if self._min_resolution is not None:
( min_width, min_height ) = self._min_resolution
too_thin = width is not None and width < min_width
too_short = height is not None and height < min_height
if too_thin or too_short:
raise HydrusExceptions.FileSizeException( 'File had resolution ' + HydrusData.ConvertResolutionToPrettyString( ( width, height ) ) + ' but the lower limit is ' + HydrusData.ConvertResolutionToPrettyString( self._min_resolution ) )
if self._max_resolution is not None:
( max_width, max_height ) = self._max_resolution
too_wide = width is not None and width > max_width
too_tall = height is not None and height > max_height
if too_wide or too_tall:
raise HydrusExceptions.FileSizeException( 'File had resolution ' + HydrusData.ConvertResolutionToPrettyString( ( width, height ) ) + ' but the upper limit is ' + HydrusData.ConvertResolutionToPrettyString( self._max_resolution ) )
def _ConvertRuleToListCtrlTuples(self, rule):
(bandwidth_type, time_delta, max_allowed) = rule
pretty_time_delta = HydrusData.TimeDeltaToPrettyTimeDelta(time_delta)
if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
pretty_max_allowed = HydrusData.ToHumanBytes(max_allowed)
elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
pretty_max_allowed = HydrusData.ToHumanInt(
max_allowed) + ' requests'
sort_time_delta = ClientGUIListCtrl.SafeNoneInt(time_delta)
sort_tuple = (max_allowed, sort_time_delta)
display_tuple = (pretty_max_allowed, pretty_time_delta)
return (display_tuple, sort_tuple)
def _RenderItemName(self, name, size):
return name + ' - ' + HydrusData.ToHumanBytes(size)
def _Update( self ):
if self._network_job is None or self._network_job.NoEngineYet():
can_cancel = False
self._left_text.clear()
self._right_text.clear()
self._gauge.SetRange( 1 )
self._gauge.SetValue( 0 )
else:
can_cancel = not self._network_job.IsDone()
( status_text, current_speed, bytes_read, bytes_to_read ) = self._network_job.GetStatus()
self._left_text.setText( status_text )
speed_text = ''
if bytes_read is not None and bytes_read > 0 and not self._network_job.HasError():
if bytes_to_read is not None and bytes_read != bytes_to_read:
speed_text += HydrusData.ConvertValueRangeToBytes( bytes_read, bytes_to_read )
else:
speed_text += HydrusData.ToHumanBytes( bytes_read )
if current_speed != bytes_to_read: # if it is a real quick download, just say its size
speed_text += ' ' + HydrusData.ToHumanBytes( current_speed ) + '/s'
show_right_text = speed_text != ''
if self._right_text.isVisible() != show_right_text:
self._right_text.setVisible( show_right_text )
self.updateGeometry()
if speed_text != '':
self._right_text.setText( speed_text )
right_width = ClientGUIFunctions.ConvertTextToPixelWidth( self._right_text, len( speed_text ) )
right_min_width = right_width
if right_min_width != self._last_right_min_width:
self._last_right_min_width = right_min_width
self._right_text.setMinimumWidth( right_min_width )
self.updateGeometry()
self._gauge.SetRange( bytes_to_read )
self._gauge.SetValue( bytes_read )
if self._cancel_button.isEnabled() != can_cancel:
self._cancel_button.setEnabled( can_cancel )
def __init__( self, parent, payload_obj, title = None, description = None, payload_description = None ):
ClientGUIScrolledPanels.ReviewPanel.__init__( self, parent )
self._payload_obj = payload_obj
self._filepicker = QP.FilePickerCtrl( self, wildcard = 'PNG (*.png)' )
self._filepicker.SetSaveMode( True )
flp_width = ClientGUIFunctions.ConvertTextToPixelWidth( self._filepicker, 64 )
self._filepicker.setMinimumWidth( flp_width )
self._title = QW.QLineEdit( self )
self._payload_description = QW.QLineEdit( self )
self._text = QW.QLineEdit( self )
self._width = QP.MakeQSpinBox( self, min=100, max=4096 )
self._export = ClientGUICommon.BetterButton( self, 'export', self.Export )
#
if payload_description is None:
( payload_description, payload_bytes ) = ClientSerialisable.GetPayloadDescriptionAndBytes( self._payload_obj )
else:
payload_bytes = ClientSerialisable.GetPayloadBytes( self._payload_obj )
payload_description += ' - ' + HydrusData.ToHumanBytes( len( payload_bytes ) )
self._payload_description.setText( payload_description )
self._payload_description.setEnabled( False )
self._width.setValue( 512 )
last_png_export_dir = HG.client_controller.new_options.GetNoneableString( 'last_png_export_dir' )
if title is not None:
name = title
elif isinstance( self._payload_obj, HydrusSerialisable.SerialisableBaseNamed ):
name = self._payload_obj.GetName()
else:
name = payload_description
self._title.setText( name )
if description is not None:
self._text.setText( description )
if last_png_export_dir is not None:
filename = name + '.png'
filename = HydrusPaths.SanitizeFilename( filename )
path = os.path.join( last_png_export_dir, filename )
self._filepicker.SetPath( path )
self._Update()
#
rows = []
rows.append( ( 'export path: ', self._filepicker ) )
rows.append( ( 'title: ', self._title ) )
rows.append( ( 'payload description: ', self._payload_description ) )
rows.append( ( 'your description (optional): ', self._text ) )
rows.append( ( 'png width: ', self._width ) )
rows.append( ( '', self._export ) )
gridbox = ClientGUICommon.WrapInGrid( self, rows )
self.widget().setLayout( gridbox )
self._filepicker.filePickerChanged.connect( self._Update )
self._title.textChanged.connect( self._Update )
def _Update( self ):
if self._network_job is None or self._network_job.NoEngineYet():
self._left_text.setText( '' )
self._right_text.setText( '' )
self._gauge.SetRange( 1 )
self._gauge.SetValue( 0 )
can_cancel = False
else:
if self._network_job.IsDone():
can_cancel = False
else:
can_cancel = True
( status_text, current_speed, bytes_read, bytes_to_read ) = self._network_job.GetStatus()
self._left_text.setText( status_text )
if not self._download_started and current_speed > 0:
self._download_started = True
speed_text = ''
if self._download_started and not self._network_job.HasError():
if bytes_read is not None:
if bytes_to_read is not None and bytes_read != bytes_to_read:
speed_text += HydrusData.ConvertValueRangeToBytes( bytes_read, bytes_to_read )
else:
speed_text += HydrusData.ToHumanBytes( bytes_read )
if current_speed != bytes_to_read: # if it is a real quick download, just say its size
speed_text += ' ' + HydrusData.ToHumanBytes( current_speed ) + '/s'
self._right_text.setText( speed_text )
right_width = ClientGUIFunctions.ConvertTextToPixelWidth( self._right_text, len( speed_text ) )
right_min_width = right_width
if right_min_width != self._last_right_min_width:
self._last_right_min_width = right_min_width
self._right_text.setMinimumWidth( right_min_width )
self._gauge.SetRange( bytes_to_read )
self._gauge.SetValue( bytes_read )
if can_cancel:
if not self._cancel_button.isEnabled():
self._cancel_button.setEnabled( True )
else:
if self._cancel_button.isEnabled():
self._cancel_button.setEnabled( False )
def CheckHasSpaceForDBTransaction(db_dir, num_bytes):
if HG.no_db_temp_files:
space_needed = int(num_bytes * 1.1)
approx_available_memory = psutil.virtual_memory().available * 4 / 5
if approx_available_memory < num_bytes:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' available memory, since you are running in no_db_temp_files mode, but you only seem to have '
+ HydrusData.ToHumanBytes(approx_available_memory) + '.')
db_disk_free_space = GetFreeSpace(db_dir)
if db_disk_free_space < space_needed:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' on your db\'s partition, but you only seem to have ' +
HydrusData.ToHumanBytes(db_disk_free_space) + '.')
else:
temp_dir = tempfile.gettempdir()
temp_disk_free_space = GetFreeSpace(temp_dir)
temp_and_db_on_same_device = GetDevice(temp_dir) == GetDevice(db_dir)
if temp_and_db_on_same_device:
space_needed = int(num_bytes * 2.2)
if temp_disk_free_space < space_needed:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' on your db\'s partition, which I think also holds your temporary path, but you only seem to have '
+ HydrusData.ToHumanBytes(temp_disk_free_space) + '.')
else:
space_needed = int(num_bytes * 1.1)
if temp_disk_free_space < space_needed:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' on your temporary path\'s partition, which I think is ' +
temp_dir + ', but you only seem to have ' +
HydrusData.ToHumanBytes(temp_disk_free_space) + '.')
db_disk_free_space = GetFreeSpace(db_dir)
if db_disk_free_space < space_needed:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' on your db\'s partition, but you only seem to have ' +
HydrusData.ToHumanBytes(db_disk_free_space) + '.')
def SetJSONDump(self, obj, force_timestamp=None):
if isinstance(obj, HydrusSerialisable.SerialisableBaseNamed):
(dump_type, dump_name, version,
serialisable_info) = obj.GetSerialisableTuple()
store_backups = False
backup_depth = 1
if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_GUI_SESSION_CONTAINER:
if not obj.HasAllPageData():
raise Exception(
'A session with name "{}" was set to save, but it did not have all its page data!'
.format(dump_name))
hashes_to_page_data = obj.GetHashesToPageData()
self.SetHashedJSONDumps(hashes_to_page_data)
if force_timestamp is None:
store_backups = True
backup_depth = HG.client_controller.new_options.GetInteger(
'number_of_gui_session_backups')
try:
dump = json.dumps(serialisable_info)
except Exception as e:
HydrusData.ShowException(e)
HydrusData.Print(obj)
HydrusData.Print(serialisable_info)
raise Exception(
'Trying to json dump the object ' + str(obj) +
' with name ' + dump_name +
' caused an error. Its serialisable info has been dumped to the log.'
)
if force_timestamp is None:
object_timestamp = HydrusData.GetNow()
if store_backups:
existing_timestamps = sorted(
self._STI(
self._c.execute(
'SELECT timestamp FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;',
(dump_type, dump_name))))
if len(existing_timestamps) > 0:
# the user has changed their system clock, so let's make sure the new timestamp is larger at least
largest_existing_timestamp = max(existing_timestamps)
if largest_existing_timestamp > object_timestamp:
object_timestamp = largest_existing_timestamp + 1
deletee_timestamps = existing_timestamps[:
-backup_depth] # keep highest n values
deletee_timestamps.append(
object_timestamp
) # if save gets spammed twice in one second, we'll overwrite
self._c.executemany(
'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;',
[(dump_type, dump_name, timestamp)
for timestamp in deletee_timestamps])
else:
self._c.execute(
'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;',
(dump_type, dump_name))
else:
object_timestamp = force_timestamp
dump_buffer = GenerateBigSQLiteDumpBuffer(dump)
try:
self._c.execute(
'INSERT INTO json_dumps_named ( dump_type, dump_name, version, timestamp, dump ) VALUES ( ?, ?, ?, ?, ? );',
(dump_type, dump_name, version, object_timestamp,
dump_buffer))
except:
HydrusData.DebugPrint(dump)
HydrusData.ShowText(
'Had a problem saving a JSON object. The dump has been printed to the log.'
)
try:
HydrusData.Print('Dump had length {}!'.format(
HydrusData.ToHumanBytes(len(dump_buffer))))
except:
pass
raise
else:
(dump_type, version,
serialisable_info) = obj.GetSerialisableTuple()
if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER:
deletee_session_names = obj.GetDeleteeSessionNames()
dirty_session_containers = obj.GetDirtySessionContainers()
if len(deletee_session_names) > 0:
for deletee_session_name in deletee_session_names:
self.DeleteJSONDumpNamed(
HydrusSerialisable.
SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER,
dump_name=deletee_session_name)
if len(dirty_session_containers) > 0:
for dirty_session_container in dirty_session_containers:
self.SetJSONDump(dirty_session_container)
if not obj.IsDirty():
return
elif dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER:
deletee_tracker_names = obj.GetDeleteeTrackerNames()
dirty_tracker_containers = obj.GetDirtyTrackerContainers()
if len(deletee_tracker_names) > 0:
for deletee_tracker_name in deletee_tracker_names:
self.DeleteJSONDumpNamed(
HydrusSerialisable.
SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER_TRACKER_CONTAINER,
dump_name=deletee_tracker_name)
if len(dirty_tracker_containers) > 0:
for dirty_tracker_container in dirty_tracker_containers:
self.SetJSONDump(dirty_tracker_container)
if not obj.IsDirty():
return
try:
dump = json.dumps(serialisable_info)
except Exception as e:
HydrusData.ShowException(e)
HydrusData.Print(obj)
HydrusData.Print(serialisable_info)
raise Exception(
'Trying to json dump the object ' + str(obj) +
' caused an error. Its serialisable info has been dumped to the log.'
)
self._c.execute('DELETE FROM json_dumps WHERE dump_type = ?;',
(dump_type, ))
dump_buffer = GenerateBigSQLiteDumpBuffer(dump)
try:
self._c.execute(
'INSERT INTO json_dumps ( dump_type, version, dump ) VALUES ( ?, ?, ? );',
(dump_type, version, dump_buffer))
except:
HydrusData.DebugPrint(dump)
HydrusData.ShowText(
'Had a problem saving a JSON object. The dump has been printed to the log.'
)
raise
def GetSummary(self):
statements = []
if self._exclude_deleted:
statements.append('excluding previously deleted')
if not self._allow_decompression_bombs:
statements.append('excluding decompression bombs')
if self._min_size is not None:
statements.append('excluding < ' +
HydrusData.ToHumanBytes(self._min_size))
if self._max_size is not None:
statements.append('excluding > ' +
HydrusData.ToHumanBytes(self._max_size))
if self._max_gif_size is not None:
statements.append('excluding gifs > ' +
HydrusData.ToHumanBytes(self._max_gif_size))
if self._min_resolution is not None:
(width, height) = self._min_resolution
statements.append('excluding < ( ' + HydrusData.ToHumanInt(width) +
' x ' + HydrusData.ToHumanInt(height) + ' )')
if self._max_resolution is not None:
(width, height) = self._max_resolution
statements.append('excluding > ( ' + HydrusData.ToHumanInt(width) +
' x ' + HydrusData.ToHumanInt(height) + ' )')
#
if self._automatic_archive:
statements.append('automatically archiving')
#
presentation_statements = []
if self._present_new_files:
presentation_statements.append('new')
if self._present_already_in_inbox_files:
presentation_statements.append('already in inbox')
if self._present_already_in_archive_files:
presentation_statements.append('already in archive')
if len(presentation_statements) == 0:
statements.append('not presenting any files')
elif len(presentation_statements) == 3:
statements.append('presenting all files')
else:
statements.append('presenting ' +
', '.join(presentation_statements) + ' files')
summary = os.linesep.join(statements)
return summary
