def ConvertToPNGIfBMP(path):
with open(path, 'rb') as f:
header = f.read(2)
if header == b'BM':
(os_file_handle, temp_path) = HydrusTemp.GetTempPath()
try:
with open(path, 'rb') as f_source:
with open(temp_path, 'wb') as f_dest:
HydrusPaths.CopyFileLikeToFileLike(f_source, f_dest)
pil_image = GeneratePILImage(temp_path)
pil_image.save(path, 'PNG')
finally:
HydrusTemp.CleanUpTempPath(os_file_handle, temp_path)
def boot():
if result.temp_dir is not None:
HydrusTemp.SetEnvTempDir(result.temp_dir)
controller = None
with HydrusLogger.HydrusLogger(db_dir, 'client') as logger:
try:
HydrusData.Print('hydrus client started')
if not HG.twisted_is_broke:
import threading
threading.Thread(target=reactor.run,
name='twisted',
kwargs={
'installSignalHandlers': 0
}).start()
from hydrus.client import ClientController
controller = ClientController.Controller(db_dir)
controller.Run()
except:
HydrusData.Print('hydrus client failed')
import traceback
HydrusData.Print(traceback.format_exc())
finally:
HG.started_shutdown = True
HG.view_shutdown = True
HG.model_shutdown = True
if controller is not None:
controller.pubimmediate('wake_daemons')
if not HG.twisted_is_broke:
reactor.callFromThread(reactor.stop)
HydrusData.Print('hydrus client shut down')
HG.shutdown_complete = True
if HG.restart:
HydrusData.RestartProcess()
def DumpToPNG(width, payload_bytes, title, payload_description, text, path):
payload_bytes_length = len(payload_bytes)
header_and_payload_bytes_length = payload_bytes_length + 4
payload_height = int(header_and_payload_bytes_length / width)
if (header_and_payload_bytes_length / width) % 1.0 > 0:
payload_height += 1
top_image = CreateTopImage(width, title, payload_description, text)
payload_length_header = struct.pack('!I', payload_bytes_length)
num_empty_bytes = payload_height * width - header_and_payload_bytes_length
header_and_payload_bytes = payload_length_header + payload_bytes + b'\x00' * num_empty_bytes
payload_image = numpy.fromstring(header_and_payload_bytes,
dtype='uint8').reshape(
(payload_height, width))
finished_image = numpy.concatenate((top_image, payload_image))
# this is to deal with unicode paths, which cv2 can't handle
(os_file_handle, temp_path) = HydrusTemp.GetTempPath(suffix='.png')
try:
cv2.imwrite(temp_path, finished_image,
[cv2.IMWRITE_PNG_COMPRESSION, 9])
HydrusPaths.MirrorFile(temp_path, path)
except Exception as e:
HydrusData.ShowException(e)
raise Exception('Could not save the png!')
finally:
HydrusTemp.CleanUpTempPath(os_file_handle, temp_path)
def test_import_folders_daemon( self ):
test_dir = HydrusTemp.GetTempDir()
try:
HG.test_controller.SetRead( 'hash_status', ClientImportFiles.FileImportStatus.STATICGetUnknownStatus() )
HydrusPaths.MakeSureDirectoryExists( test_dir )
hydrus_png_path = os.path.join( HC.STATIC_DIR, 'hydrus.png' )
HydrusPaths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '0' ) )
HydrusPaths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '1' ) ) # previously imported
HydrusPaths.MirrorFile( hydrus_png_path, os.path.join( test_dir, '2' ) )
with open( os.path.join( test_dir, '3' ), 'wb' ) as f: f.write( b'blarg' ) # broken
with open( os.path.join( test_dir, '4' ), 'wb' ) as f: f.write( b'blarg' ) # previously failed
#
actions = {}
actions[ CC.STATUS_SUCCESSFUL_AND_NEW ] = CC.IMPORT_FOLDER_DELETE
actions[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ] = CC.IMPORT_FOLDER_DELETE
actions[ CC.STATUS_DELETED ] = CC.IMPORT_FOLDER_DELETE
actions[ CC.STATUS_ERROR ] = CC.IMPORT_FOLDER_IGNORE
import_folder = ClientImportLocal.ImportFolder( 'imp', path = test_dir, actions = actions )
HG.test_controller.SetRead( 'serialisable_names', [ 'imp' ] )
HG.test_controller.SetRead( 'serialisable_named', import_folder )
HG.test_controller.ClearWrites( 'import_file' )
HG.test_controller.ClearWrites( 'serialisable' )
ClientDaemons.DAEMONCheckImportFolders()
import_file = HG.test_controller.GetWrite( 'import_file' )
self.assertEqual( len( import_file ), 3 )
# I need to expand tests here with the new file system
[ ( ( updated_import_folder, ), empty_dict ) ] = HG.test_controller.GetWrite( 'serialisable' )
self.assertEqual( updated_import_folder, import_folder )
self.assertTrue( not os.path.exists( os.path.join( test_dir, '0' ) ) )
self.assertTrue( not os.path.exists( os.path.join( test_dir, '1' ) ) )
self.assertTrue( not os.path.exists( os.path.join( test_dir, '2' ) ) )
self.assertTrue( os.path.exists( os.path.join( test_dir, '3' ) ) )
self.assertTrue( os.path.exists( os.path.join( test_dir, '4' ) ) )
finally:
shutil.rmtree( test_dir )
def LoadFromPNG( path ):
# this is to deal with unicode paths, which cv2 can't handle
( os_file_handle, temp_path ) = HydrusTemp.GetTempPath()
try:
HydrusPaths.MirrorFile( path, temp_path )
try:
# unchanged because we want exact byte data, no conversions or other gubbins
numpy_image = cv2.imread( temp_path, flags = IMREAD_UNCHANGED )
if numpy_image is None:
raise Exception()
except Exception as e:
try:
# dequantize = False because we don't want to convert to RGB
pil_image = HydrusImageHandling.GeneratePILImage( temp_path, dequantize = False )
numpy_image = HydrusImageHandling.GenerateNumPyImageFromPILImage( pil_image )
except Exception as e:
HydrusData.ShowException( e )
raise Exception( '"{}" did not appear to be a valid image!'.format( path ) )
finally:
HydrusTemp.CleanUpTempPath( os_file_handle, temp_path )
return LoadFromNumPyImage( numpy_image )
def GenerateThumbnailBytes(path,
target_resolution,
mime,
duration,
num_frames,
clip_rect=None,
percentage_in=35):
if target_resolution == (0, 0):
target_resolution = (128, 128)
if mime in (HC.IMAGE_JPEG, HC.IMAGE_PNG, HC.IMAGE_GIF, HC.IMAGE_WEBP,
HC.IMAGE_TIFF, HC.IMAGE_ICON): # not apng atm
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
path, target_resolution, mime, clip_rect=clip_rect)
elif mime == HC.APPLICATION_PSD:
(os_file_handle, temp_path) = HydrusTemp.GetTempPath(suffix='.png')
try:
HydrusVideoHandling.RenderImageToPNGPath(path, temp_path)
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
temp_path, target_resolution, mime, clip_rect=clip_rect)
except:
thumb_path = os.path.join(HC.STATIC_DIR, 'psd.png')
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
thumb_path, target_resolution, mime, clip_rect=clip_rect)
finally:
HydrusTemp.CleanUpTempPath(os_file_handle, temp_path)
elif mime == HC.APPLICATION_CLIP:
(os_file_handle, temp_path) = HydrusTemp.GetTempPath()
try:
HydrusClipHandling.ExtractDBPNGToPath(path, temp_path)
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
temp_path, target_resolution, mime, clip_rect=clip_rect)
except:
thumb_path = os.path.join(HC.STATIC_DIR, 'clip.png')
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
thumb_path, target_resolution, mime, clip_rect=clip_rect)
finally:
HydrusTemp.CleanUpTempPath(os_file_handle, temp_path)
elif mime == HC.APPLICATION_FLASH:
(os_file_handle, temp_path) = HydrusTemp.GetTempPath()
try:
HydrusFlashHandling.RenderPageToFile(path, temp_path, 1)
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
temp_path, target_resolution, mime, clip_rect=clip_rect)
except:
thumb_path = os.path.join(HC.STATIC_DIR, 'flash.png')
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
thumb_path, target_resolution, mime, clip_rect=clip_rect)
finally:
HydrusTemp.CleanUpTempPath(os_file_handle, temp_path)
else:
renderer = HydrusVideoHandling.VideoRendererFFMPEG(path,
mime,
duration,
num_frames,
target_resolution,
clip_rect=clip_rect)
renderer.read_frame(
) # this initialises the renderer and loads the first frame as a fallback
desired_thumb_frame = int((percentage_in / 100.0) * num_frames)
renderer.set_position(desired_thumb_frame)
numpy_image = renderer.read_frame()
if numpy_image is None:
raise Exception('Could not create a thumbnail from that video!')
numpy_image = HydrusImageHandling.ResizeNumPyImage(
numpy_image,
target_resolution) # just in case ffmpeg doesn't deliver right
thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesNumPy(
numpy_image, mime)
renderer.Stop()
del renderer
return thumbnail_bytes
def MaintainMemorySlow( self ):
HydrusTemp.CleanUpOldTempPaths()
self._MaintainCallToThreads()
def _InitTempDir( self ):
self.temp_dir = HydrusTemp.GetTempDir()
def MainLoop(self):
hashes_still_to_download_in_this_run = set()
total_hashes_in_this_run = 0
total_successful_hashes_in_this_run = 0
while not (HydrusThreading.IsThreadShuttingDown()
or self._shutting_down or HG.view_shutdown):
with self._lock:
if len(self._pending_hashes) > 0:
if total_hashes_in_this_run == 0:
job_key = ClientThreading.JobKey(cancellable=True)
job_key.SetStatusTitle('downloading')
job_key.SetVariable('popup_text_1',
'initialising downloader')
job_key_pub_job = self._controller.CallLater(
2.0, self._controller.pub, 'message', job_key)
num_before = len(hashes_still_to_download_in_this_run)
hashes_still_to_download_in_this_run.update(
self._pending_hashes)
num_after = len(hashes_still_to_download_in_this_run)
total_hashes_in_this_run += num_after - num_before
self._pending_hashes = set()
if len(hashes_still_to_download_in_this_run) == 0:
total_hashes_in_this_run = 0
total_successful_hashes_in_this_run = 0
self._new_files_event.wait(5)
self._new_files_event.clear()
continue
if job_key.IsCancelled():
hashes_still_to_download_in_this_run = set()
continue
hash = random.sample(hashes_still_to_download_in_this_run, 1)[0]
hashes_still_to_download_in_this_run.discard(hash)
total_done = total_hashes_in_this_run - len(
hashes_still_to_download_in_this_run)
job_key.SetVariable(
'popup_text_1',
'downloading files from remote services: {}'.format(
HydrusData.ConvertValueRangeToPrettyString(
total_done, total_hashes_in_this_run)))
job_key.SetVariable('popup_gauge_1',
(total_done, total_hashes_in_this_run))
try:
errors_occured = []
file_successful = False
media_result = self._controller.Read('media_result', hash)
service_keys = list(
media_result.GetLocationsManager().GetCurrent())
random.shuffle(service_keys)
if CC.COMBINED_LOCAL_FILE_SERVICE_KEY in service_keys:
total_successful_hashes_in_this_run += 1
continue
for service_key in service_keys:
try:
service = self._controller.services_manager.GetService(
service_key)
except:
continue
try:
if service.GetServiceType() == HC.FILE_REPOSITORY:
file_repository = service
if file_repository.IsFunctional():
(os_file_handle,
temp_path) = HydrusTemp.GetTempPath()
try:
file_repository.Request(
HC.GET,
'file', {'hash': hash},
temp_path=temp_path)
exclude_deleted = False # this is the important part here
do_not_check_known_urls_before_importing = False
do_not_check_hashes_before_importing = False
allow_decompression_bombs = True
min_size = None
max_size = None
max_gif_size = None
min_resolution = None
max_resolution = None
automatic_archive = False
associate_primary_urls = True
associate_source_urls = True
file_import_options = FileImportOptions.FileImportOptions(
)
file_import_options.SetPreImportOptions(
exclude_deleted,
do_not_check_known_urls_before_importing,
do_not_check_hashes_before_importing,
allow_decompression_bombs, min_size,
max_size, max_gif_size, min_resolution,
max_resolution)
file_import_options.SetPostImportOptions(
automatic_archive,
associate_primary_urls,
associate_source_urls)
file_import_job = ClientImportFiles.FileImportJob(
temp_path, file_import_options)
file_import_job.DoWork()
file_successful = True
break
finally:
HydrusTemp.CleanUpTempPath(
os_file_handle, temp_path)
elif service.GetServiceType() == HC.IPFS:
multihashes = HG.client_controller.Read(
'service_filenames', service_key, {hash})
if len(multihashes) > 0:
multihash = multihashes[0]
service.ImportFile(multihash, silent=True)
file_successful = True
break
except Exception as e:
errors_occured.append(e)
if file_successful:
total_successful_hashes_in_this_run += 1
if len(errors_occured) > 0:
if not file_successful:
raise errors_occured[0]
except Exception as e:
HydrusData.ShowException(e)
hashes_still_to_download_in_this_run = 0
finally:
if len(hashes_still_to_download_in_this_run) == 0:
job_key.DeleteVariable('popup_text_1')
job_key.DeleteVariable('popup_gauge_1')
if total_successful_hashes_in_this_run > 0:
job_key.SetVariable(
'popup_text_1',
HydrusData.ToHumanInt(
total_successful_hashes_in_this_run) +
' files downloaded')
job_key_pub_job.Cancel()
job_key.Finish()
job_key.Delete(1)
def LoadFromPNG(path):
# this is to deal with unicode paths, which cv2 can't handle
(os_file_handle, temp_path) = HydrusTemp.GetTempPath()
try:
HydrusPaths.MirrorFile(path, temp_path)
try:
# unchanged because we want exact byte data, no conversions or other gubbins
numpy_image = cv2.imread(temp_path, flags=IMREAD_UNCHANGED)
if numpy_image is None:
raise Exception()
except Exception as e:
try:
# dequantize = False because we don't want to convert to RGB
pil_image = HydrusImageHandling.GeneratePILImage(
temp_path, dequantize=False)
numpy_image = HydrusImageHandling.GenerateNumPyImageFromPILImage(
pil_image)
except Exception as e:
HydrusData.ShowException(e)
raise Exception('That did not appear to be a valid image!')
finally:
HydrusTemp.CleanUpTempPath(os_file_handle, temp_path)
try:
height = numpy_image.shape[0]
width = numpy_image.shape[1]
if len(numpy_image.shape) > 2:
depth = numpy_image.shape[2]
if depth != 1:
raise Exception('The file did not appear to be monochrome!')
try:
complete_data = numpy_image.tostring()
top_height_header = complete_data[:2]
(top_height, ) = struct.unpack('!H', top_height_header)
payload_and_header_bytes = complete_data[width * top_height:]
except:
raise Exception('Header bytes were invalid!')
try:
payload_length_header = payload_and_header_bytes[:4]
(payload_bytes_length, ) = struct.unpack('!I',
payload_length_header)
payload_bytes = payload_and_header_bytes[4:4 +
payload_bytes_length]
except:
raise Exception('Payload bytes were invalid!')
except Exception as e:
HydrusData.PrintException(e)
message = 'The image loaded, but it did not seem to be a hydrus serialised png! The error was: {}'.format(
str(e))
message += os.linesep * 2
message += 'If you believe this is a legit non-resized, non-converted hydrus serialised png, please send it to hydrus_dev.'
raise Exception(message)
return payload_bytes
if HG.db_journal_mode == 'WAL':
HG.db_synchronous = 1
else:
HG.db_synchronous = 2
HG.no_db_temp_files = result.no_db_temp_files
HG.boot_debug = result.boot_debug
if result.temp_dir is not None:
HydrusTemp.SetEnvTempDir(result.temp_dir)
except Exception as e:
import traceback
error_trace = traceback.format_exc()
print(error_trace)
if 'db_dir' in locals() and os.path.exists(db_dir):
emergency_dir = db_dir
else:
def _callbackParsePOSTArgs(self,
request: HydrusServerRequest.HydrusRequest):
request.content.seek(0)
if not request.requestHeaders.hasHeader('Content-Type'):
parsed_request_args = HydrusNetworkVariableHandling.ParsedRequestArguments(
)
else:
content_types = request.requestHeaders.getRawHeaders(
'Content-Type')
content_type = content_types[0]
try:
mime = HC.mime_enum_lookup[content_type]
except:
raise HydrusExceptions.BadRequestException(
'Did not recognise Content-Type header!')
total_bytes_read = 0
if mime == HC.APPLICATION_JSON:
json_string = request.content.read()
total_bytes_read += len(json_string)
parsed_request_args = HydrusNetworkVariableHandling.ParseNetworkBytesToParsedHydrusArgs(
json_string)
else:
(os_file_handle, temp_path) = HydrusTemp.GetTempPath()
request.temp_file_info = (os_file_handle, temp_path)
with open(temp_path, 'wb') as f:
for block in HydrusPaths.ReadFileLikeAsBlocks(
request.content):
f.write(block)
total_bytes_read += len(block)
decompression_bombs_ok = self._DecompressionBombsOK(request)
parsed_request_args = HydrusNetworkVariableHandling.ParseFileArguments(
temp_path, decompression_bombs_ok)
self._reportDataUsed(request, total_bytes_read)
request.parsed_request_args = parsed_request_args
return request
def CheckHasSpaceForDBTransaction(db_dir, num_bytes):
if HG.no_db_temp_files:
space_needed = int(num_bytes * 1.1)
approx_available_memory = psutil.virtual_memory().available * 4 / 5
if approx_available_memory < num_bytes:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' available memory, since you are running in no_db_temp_files mode, but you only seem to have '
+ HydrusData.ToHumanBytes(approx_available_memory) + '.')
db_disk_free_space = HydrusPaths.GetFreeSpace(db_dir)
if db_disk_free_space < space_needed:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' on your db\'s disk partition, but you only seem to have ' +
HydrusData.ToHumanBytes(db_disk_free_space) + '.')
else:
temp_dir = HydrusTemp.GetCurrentTempDir()
temp_disk_free_space = HydrusPaths.GetFreeSpace(temp_dir)
temp_and_db_on_same_device = HydrusPaths.GetDevice(
temp_dir) == HydrusPaths.GetDevice(db_dir)
if temp_and_db_on_same_device:
space_needed = int(num_bytes * 2.2)
if temp_disk_free_space < space_needed:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' on your db\'s disk partition, which I think also holds your temporary path, but you only seem to have '
+ HydrusData.ToHumanBytes(temp_disk_free_space) + '.')
else:
space_needed = int(num_bytes * 1.1)
if temp_disk_free_space < space_needed:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' on your temporary path\'s disk partition, which I think is '
+ temp_dir + ', but you only seem to have ' +
HydrusData.ToHumanBytes(temp_disk_free_space) + '.')
db_disk_free_space = HydrusPaths.GetFreeSpace(db_dir)
if db_disk_free_space < space_needed:
raise Exception(
'I believe you need about ' +
HydrusData.ToHumanBytes(space_needed) +
' on your db\'s disk partition, but you only seem to have '
+ HydrusData.ToHumanBytes(db_disk_free_space) + '.')