def SetServices( self, services ):
# doesn't need the dirty_object_lock because the caller takes it
# first test available ports
my_ports = { s.GetPort() for s in self._services }
for service in services:
port = service.GetPort()
if port not in my_ports and HydrusNetworking.LocalPortInUse( port ):
raise HydrusExceptions.ServerException( 'Something was already bound to port ' + str( port ) )
#
self._services = services
self.CallToThread( self.services_upnp_manager.SetServices, self._services )
[ self._admin_service ] = [ service for service in self._services if service.GetServiceType() == HC.SERVER_ADMIN ]
self.SetRunningTwistedServices( self._services )
def GetTagId( self, tag ) -> int:
clean_tag = HydrusTags.CleanTag( tag )
try:
HydrusTags.CheckTagNotEmpty( clean_tag )
except HydrusExceptions.TagSizeException:
raise HydrusExceptions.TagSizeException( '"{}" tag seems not valid--when cleaned, it ends up with zero size!'.format( tag ) )
result = self._c.execute( 'SELECT tag_id FROM local_tags_cache WHERE tag = ?;', ( tag, ) ).fetchone()
if result is None:
return self.modules_tags.GetTagId( tag )
else:
( tag_id, ) = result
return tag_id
def GetValue(self):
service_key = self._service_keys.GetValue()
if service_key is None:
raise HydrusExceptions.VetoException(
'Please select a rating service!')
action = self._flip_or_set_action.GetValue()
if self._ratings_numerical_remove.isChecked():
rating = None
else:
value = self._ratings_numerical_slider.GetValue()
rating = self._current_ratings_numerical_service.ConvertStarsToRating(
value)
return CAC.ApplicationCommand(
CAC.APPLICATION_COMMAND_TYPE_CONTENT,
(service_key, HC.CONTENT_TYPE_RATINGS, action, rating))
def ParseFFMPEGVideoLine(lines, png_ok=False):
if png_ok:
bad_video_formats = ['jpg']
else:
bad_video_formats = ['png', 'jpg']
# get the output line that speaks about video
# the ^\sStream is to exclude the 'title' line, when it exists, includes the string 'Video: ', ha ha
lines_video = [
l for l in lines
if re.search(r'^\s*Stream', l) is not None and 'Video: ' in l
and True not in ('Video: {}'.format(bad_video_format) in l
for bad_video_format in bad_video_formats)
] # mp3 says it has a 'png' video stream
if len(lines_video) == 0:
raise HydrusExceptions.DamagedOrUnusualFileException(
'Could not find video information!')
line = lines_video[0]
return line
def GetFFMPEGAPNGProperties(path):
with open(path, 'rb') as f:
file_header_bytes = f.read(256)
apng_actl_bytes = GetAPNGACTLChunk(file_header_bytes)
if apng_actl_bytes is None:
raise HydrusExceptions.DamagedOrUnusualFileException(
'This APNG had an unusual file header!')
num_frames = GetAPNGNumFrames(apng_actl_bytes)
lines = GetFFMPEGInfoLines(path)
resolution = ParseFFMPEGVideoResolution(lines, png_ok=True)
(fps, confident_fps) = ParseFFMPEGFPS(lines, png_ok=True)
if not confident_fps:
fps = 24
duration = num_frames / fps
duration_in_ms = int(duration * 1000)
has_audio = False
return (resolution, duration_in_ms, num_frames, has_audio)
def GetResult( self ):
time.sleep( 0.00001 ) # this one neat trick can save hassle on superquick jobs as event.wait can be laggy
while True:
if self._result_ready.wait( 2 ) == True:
break
elif HG.model_shutdown:
raise HydrusExceptions.ShutdownException( 'Application quit before db could serve result!' )
self._DoDelayedResultRelief()
if isinstance( self._result, Exception ):
e = self._result
raise e
else:
return self._result
def _GetInfo(self, share_key):
try:
info = self._keys_to_infos[share_key]
except:
raise HydrusExceptions.NotFoundException(
'Did not find that share on this booru.')
if info is None:
info = self._controller.Read('local_booru_share', share_key)
hashes = info['hashes']
info['hashes_set'] = set(hashes)
media_results = self._controller.Read('media_results', hashes)
info['media_results'] = media_results
hashes_to_media_results = {
media_result.GetHash(): media_result
for media_result in media_results
}
info['hashes_to_media_results'] = hashes_to_media_results
self._keys_to_infos[share_key] = info
return info
def _callbackParsePOSTArgs( self, request: HydrusServerRequest.HydrusRequest ):
request.content.seek( 0 )
if not request.requestHeaders.hasHeader( 'Content-Type' ):
parsed_request_args = HydrusNetworkVariableHandling.ParsedRequestArguments()
else:
content_types = request.requestHeaders.getRawHeaders( 'Content-Type' )
content_type = content_types[0]
try:
mime = HC.mime_enum_lookup[ content_type ]
except:
raise HydrusExceptions.BadRequestException( 'Did not recognise Content-Type header!' )
total_bytes_read = 0
if mime == HC.APPLICATION_JSON:
json_string = request.content.read()
total_bytes_read += len( json_string )
parsed_request_args = HydrusNetworkVariableHandling.ParseNetworkBytesToParsedHydrusArgs( json_string )
else:
( os_file_handle, temp_path ) = HydrusPaths.GetTempPath()
request.temp_file_info = ( os_file_handle, temp_path )
with open( temp_path, 'wb' ) as f:
for block in HydrusPaths.ReadFileLikeAsBlocks( request.content ):
f.write( block )
total_bytes_read += len( block )
decompression_bombs_ok = self._DecompressionBombsOK( request )
parsed_request_args = HydrusNetworkVariableHandling.ParseFileArguments( temp_path, decompression_bombs_ok )
self._reportDataUsed( request, total_bytes_read )
request.parsed_request_args = parsed_request_args
return request
def CheckTagNotEmpty(tag):
(namespace, subtag) = SplitTag(tag)
if subtag == '':
raise HydrusExceptions.TagSizeException('Received a zero-length tag!')
def GetYAMLDump( self, dump_type, dump_name = None ):
if dump_name is None:
result = { dump_name : data for ( dump_name, data ) in self._c.execute( 'SELECT dump_name, dump FROM yaml_dumps WHERE dump_type = ?;', ( dump_type, ) ) }
if dump_type == YAML_DUMP_ID_LOCAL_BOORU:
result = { bytes.fromhex( dump_name ) : data for ( dump_name, data ) in list(result.items()) }
else:
if dump_type == YAML_DUMP_ID_LOCAL_BOORU: dump_name = dump_name.hex()
result = self._c.execute( 'SELECT dump FROM yaml_dumps WHERE dump_type = ? AND dump_name = ?;', ( dump_type, dump_name ) ).fetchone()
if result is None:
if result is None:
raise HydrusExceptions.DataMissing( dump_name + ' was not found!' )
else:
( result, ) = result
return result
def _threadDoPOSTJob(self, request: HydrusServerRequest.HydrusRequest):
services = request.parsed_request_args['services']
unique_ports = {service.GetPort() for service in services}
if len(unique_ports) < len(services):
raise HydrusExceptions.BadRequestException(
'It looks like some of those services share ports! Please give them unique ports!'
)
with HG.dirty_object_lock:
HG.server_controller.SetServices(services)
service_keys_to_access_keys = HG.server_controller.WriteSynchronous(
'services', request.hydrus_account, services)
body = HydrusNetworkVariableHandling.DumpHydrusArgsToNetworkBytes(
{'service_keys_to_access_keys': service_keys_to_access_keys})
response_context = HydrusServerResources.ResponseContext(200,
body=body)
return response_context
def InitialiseFromSerialisableInfo(self,
version,
serialisable_info,
raise_error_on_future_version=False):
if version > self.SERIALISABLE_VERSION:
if raise_error_on_future_version:
message = 'Unfortunately, an object of type {} could not be loaded because it was created in a client that uses an updated version of that object! This client supports versions up to {}, but the object was version {}.'.format(
self.SERIALISABLE_NAME, self.SERIALISABLE_VERSION, version)
message += os.linesep * 2
message += 'Please update your client to import this object.'
raise HydrusExceptions.SerialisationException(message)
else:
message = 'An object of type {} was created in a client that uses an updated version of that object! This client supports versions up to {}, but the object was version {}. For now, the client will try to continue work, but things may break. If you know why this has occured, please correct it. If you do not, please let hydrus dev know.'.format(
self.SERIALISABLE_NAME, self.SERIALISABLE_VERSION, version)
HydrusData.ShowText(message)
while version < self.SERIALISABLE_VERSION:
(version, serialisable_info) = self._UpdateSerialisableInfo(
version, serialisable_info)
self._InitialiseFromSerialisableInfo(serialisable_info)
def DealWithBrokenJSONDump( db_dir, dump, dump_descriptor ):
timestamp_string = time.strftime( '%Y-%m-%d %H-%M-%S' )
hex_chars = os.urandom( 4 ).hex()
filename = '({}) at {} {}.json'.format( dump_descriptor, timestamp_string, hex_chars )
path = os.path.join( db_dir, filename )
with open( path, 'wb' ) as f:
if isinstance( dump, str ):
dump = bytes( dump, 'utf-8', errors = 'replace' )
f.write( dump )
message = 'A serialised object failed to load! Its description is "{}".'.format( dump_descriptor )
message += os.linesep * 2
message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).'
message += os.linesep * 2
message += 'The database has attempted to delete the broken object, errors have been written to the log, and the object\'s dump written to {}. Depending on the object, your client may no longer be able to boot, or it may have lost something like a session or a subscription.'.format( path )
message += os.linesep * 2
message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.'
HydrusData.ShowText( message )
raise HydrusExceptions.SerialisationException( message )
def Read(self, name, *args, **kwargs):
self._read_call_args[name].append((args, kwargs))
if self._test_db is not None:
return self._test_db.Read(name, *args, **kwargs)
try:
if (name, args) in self._param_read_responses:
return self._param_read_responses[(name, args)]
except:
pass
result = self._name_read_responses[name]
if isinstance(result, Exception):
raise HydrusExceptions.DBException(result, str(result),
'test trace')
return result
def GetDeleteFilesJobs( win, media, default_reason, suggested_file_service_key = None ):
title = 'Delete files?'
with ClientGUITopLevelWindowsPanels.DialogEdit( win, title, frame_key = 'regular_center_dialog' ) as dlg:
panel = ClientGUIScrolledPanelsEdit.EditDeleteFilesPanel( dlg, media, default_reason, suggested_file_service_key = suggested_file_service_key )
dlg.SetPanel( panel )
if panel.QuestionIsAlreadyResolved():
( involves_physical_delete, jobs ) = panel.GetValue()
return ( involves_physical_delete, jobs )
if dlg.exec() == QW.QDialog.Accepted:
( involves_physical_delete, jobs ) = panel.GetValue()
return ( involves_physical_delete, jobs )
else:
raise HydrusExceptions.CancelledException( 'Dialog cancelled.' )
def CheckReadyToImport(self) -> bool:
if self._import_destination_location_context.IsEmpty():
raise HydrusExceptions.FileImportBlockException(
'There is no import destination set in the File Import Options!'
)
def qt_code(win: QW.QWidget, job_key: ClientThreading.JobKey):
try:
if win is not None and not QP.isValid(win):
raise HydrusExceptions.QtDeadWindowException(
'Parent Window was destroyed before Qt command was called!'
)
result = func(*args, **kwargs)
job_key.SetVariable('result', result)
except (HydrusExceptions.QtDeadWindowException,
HydrusExceptions.DBCredentialsException,
HydrusExceptions.ShutdownException) as e:
job_key.SetErrorException(e)
except Exception as e:
job_key.SetErrorException(e)
HydrusData.Print('CallBlockingToQt just caught this error:')
HydrusData.DebugPrint(traceback.format_exc())
finally:
job_key.Finish()
def _GetListCtrlMenu( self ):
selected_file_seeds = self._list_ctrl.GetData( only_selected = True )
if len( selected_file_seeds ) == 0:
raise HydrusExceptions.DataMissing()
menu = QW.QMenu()
can_show_files_in_new_page = True in ( file_seed.HasHash() for file_seed in selected_file_seeds )
if can_show_files_in_new_page:
ClientGUIMenus.AppendMenuItem( menu, 'open selected import files in a new page', 'Show all the known selected files in a new thumbnail page. This is complicated, so cannot always be guaranteed, even if the import says \'success\'.', self._ShowSelectionInNewPage )
ClientGUIMenus.AppendSeparator( menu )
ClientGUIMenus.AppendMenuItem( menu, 'copy sources', 'Copy all the selected sources to clipboard.', self._CopySelectedFileSeedData )
ClientGUIMenus.AppendMenuItem( menu, 'copy notes', 'Copy all the selected notes to clipboard.', self._CopySelectedNotes )
ClientGUIMenus.AppendSeparator( menu )
ClientGUIMenus.AppendMenuItem( menu, 'open sources', 'Open all the selected sources in your file explorer or web browser.', self._OpenSelectedFileSeedData )
ClientGUIMenus.AppendSeparator( menu )
ClientGUIMenus.AppendMenuItem( menu, 'try again', 'Reset the progress of all the selected imports.', HydrusData.Call( self._SetSelected, CC.STATUS_UNKNOWN ) )
ClientGUIMenus.AppendMenuItem( menu, 'skip', 'Skip all the selected imports.', HydrusData.Call( self._SetSelected, CC.STATUS_SKIPPED ) )
ClientGUIMenus.AppendMenuItem( menu, 'delete from list', 'Remove all the selected imports.', self._DeleteSelected )
return menu
def _CheckDataUsage(self):
if not self._local_booru_service.BandwidthOK():
raise HydrusExceptions.InsufficientCredentialsException(
'This booru has used all its monthly data. Please try again next month.'
)
def GetValue( self ):
service_key = self._service_keys.GetValue()
if service_key is None:
raise HydrusExceptions.VetoException( 'Please select a rating service!' )
action = self._flip_or_set_action.GetValue()
if self._ratings_like_like.isChecked():
value = 1.0
elif self._ratings_like_dislike.isChecked():
value = 0.0
else:
value = None
return CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_CONTENT, ( service_key, HC.CONTENT_TYPE_RATINGS, action, value ) )
def GetTagId( self, tag ) -> int:
clean_tag = HydrusTags.CleanTag( tag )
try:
HydrusTags.CheckTagNotEmpty( clean_tag )
except HydrusExceptions.TagSizeException:
# update this to instead go 'hey, does the dirty tag exist?' if it does, run the fix invalid tags routine
raise HydrusExceptions.TagSizeException( '"{}" tag seems not valid--when cleaned, it ends up with zero size!'.format( tag ) )
( namespace, subtag ) = HydrusTags.SplitTag( clean_tag )
namespace_id = self.GetNamespaceId( namespace )
subtag_id = self.GetSubtagId( subtag )
result = self._c.execute( 'SELECT tag_id FROM tags WHERE namespace_id = ? AND subtag_id = ?;', ( namespace_id, subtag_id ) ).fetchone()
if result is None:
self._c.execute( 'INSERT INTO tags ( namespace_id, subtag_id ) VALUES ( ?, ? );', ( namespace_id, subtag_id ) )
tag_id = self._c.lastrowid
else:
( tag_id, ) = result
return tag_id
def CheckPermission(self, permission):
if not self.HasPermission(permission):
raise HydrusExceptions.InsufficientCredentialsException(
'You do not have permission to: {}'.format(
basic_permission_to_str_lookup[permission]))
def CheckCanSeeAllFiles( self ):
with self._lock:
if not ( self._HasPermission( CLIENT_API_PERMISSION_SEARCH_FILES ) and self._search_tag_filter.AllowsEverything() ):
raise HydrusExceptions.InsufficientCredentialsException( 'You do not have permission to see all files, so you cannot do this.' )
def file_service_pred_generator( o, v, u ):
if o.startswith( 'is not' ):
is_in = False
else:
is_in = True
o_dict = {
'currently in' : HC.CONTENT_STATUS_CURRENT,
'deleted from' : HC.CONTENT_STATUS_DELETED,
'pending to' : HC.CONTENT_STATUS_PENDING,
'petitioned from' : HC.CONTENT_STATUS_PETITIONED
}
status = None
for ( phrase, possible_status ) in o_dict.items():
if phrase in o:
status = possible_status
break
if status is None:
raise HydrusExceptions.BadRequestException( 'Did not understand the file service status!' )
try:
service_name = v
service_key = HG.client_controller.services_manager.GetServiceKeyFromName( HC.FILE_SERVICES, service_name )
except:
raise HydrusExceptions.BadRequestException( 'Could not find the service "{}"!'.format( service_name ) )
return ClientSearch.Predicate( ClientSearch.PREDICATE_TYPE_SYSTEM_FILE_SERVICE, ( is_in, status, service_key ) )
def ParseFFMPEGDuration( lines ):
# get duration (in seconds)
# Duration: 00:00:02.46, start: 0.033000, bitrate: 1069 kb/s
try:
# had a vid with 'Duration:' in title, ha ha, so now a regex
line = [ l for l in lines if re.search( r'^\s*Duration:', l ) is not None ][0]
if 'Duration: N/A' in line:
return ( None, None )
if 'start:' in line:
m = re.search( '(start\\: )' + '-?[0-9]+\\.[0-9]*', line )
start_offset = float( line[ m.start() + 7 : m.end() ] )
if abs( start_offset ) > 1.0: # once had a file with start offset of 957499 seconds jej
start_offset = 0
else:
start_offset = 0
match = re.search("[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9]", line)
hms = list(map(float, line[match.start()+1:match.end()].split(':')))
if len( hms ) == 1:
duration = hms[0]
elif len( hms ) == 2:
duration = 60 * hms[0] + hms[1]
elif len( hms ) ==3:
duration = 3600 * hms[0] + 60 * hms[1] + hms[2]
if duration == 0:
return ( None, None )
file_duration = duration + start_offset
stream_duration = duration
return ( file_duration, stream_duration )
except:
raise HydrusExceptions.DamagedOrUnusualFileException( 'Error reading duration!' )
def GetServiceId( self, service_key: bytes ) -> int:
if service_key in self._service_keys_to_service_ids:
return self._service_keys_to_service_ids[ service_key ]
raise HydrusExceptions.DataMissing( 'Service id error in database: key "{}" does not exist!'.format( service_key.hex() ) )
def GetPageData(self, hash) -> "GUISessionPageData":
if hash not in self._hashes_to_page_data:
raise HydrusExceptions.DataMissing(
'The page hash "{}" was not found!'.format(hash.hex()))
return self._hashes_to_page_data[hash]
def _checkService( self, request ):
if self.BLOCKED_WHEN_BUSY and HG.server_busy.locked():
raise HydrusExceptions.ServerBusyException( 'This server is busy, please try again later.' )
return request
def GetServiceType( self, service_id ) -> ClientServices.Service:
if service_id in self._service_ids_to_services:
return self._service_ids_to_services[ service_id ].GetServiceType()
raise HydrusExceptions.DataMissing( 'Service id error in database: id "{}" does not exist!'.format( service_id ) )
def _GetCurrentFrame( self ):
if self._cv_mode:
( retval, numpy_image ) = self._cv_video.read()
if not retval:
self._next_render_index = ( self._next_render_index + 1 ) % self._num_frames
raise HydrusExceptions.CantRenderWithCVException( 'CV could not render frame ' + str( self._next_render_index - 1 ) + '.' )
else:
current_frame = HydrusImageHandling.Dequantize( self._pil_image )
if current_frame.mode == 'RGBA':
if self._pil_canvas is None:
self._pil_canvas = current_frame
else:
self._pil_canvas.paste( current_frame, None, current_frame ) # use the rgba image as its own mask
elif current_frame.mode == 'RGB':
self._pil_canvas = current_frame
numpy_image = HydrusImageHandling.GenerateNumPyImageFromPILImage( self._pil_canvas )
self._next_render_index = ( self._next_render_index + 1 ) % self._num_frames
if self._next_render_index == 0:
self._RewindGIF()
else:
if not self._cv_mode:
self._pil_image.seek( self._next_render_index )
if self._pil_global_palette is not None and self._pil_image.palette == self._pil_global_palette: # for some reason, when pil falls back from local palette to global palette, a bunch of important variables reset!
self._pil_image.palette.dirty = self._pil_dirty
self._pil_image.palette.mode = self._pil_mode
self._pil_image.palette.rawmode = self._pil_rawmode
return numpy_image
