def __init__(self, paths=None, file_import_options=None, paths_to_tags=None, delete_after_success=None): HydrusSerialisable.SerialisableBase.__init__(self) if paths is None: self._file_seed_cache = None else: self._file_seed_cache = ClientImportFileSeeds.FileSeedCache() file_seeds = [] for path in paths: file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path) try: s = os.stat(path) file_seed.source_time = int(min(s.st_mtime, s.st_ctime)) except: pass file_seeds.append(file_seed) self._file_seed_cache.AddFileSeeds(file_seeds) self._file_import_options = file_import_options self._paths_to_tags = paths_to_tags self._delete_after_success = delete_after_success self._current_action = '' self._paused = False self._lock = threading.Lock() self._files_repeating_job = None HG.client_controller.sub(self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated')
def SetTuple( self, name, path, mimes, file_import_options, tag_import_options, tag_service_keys_to_filename_tagging_options, actions, action_locations, period, check_regularly, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ): if path != self._path: self._file_seed_cache = ClientImportFileSeeds.FileSeedCache() if set( mimes ) != set( self._mimes ): self._file_seed_cache.RemoveFileSeedsByStatus( ( CC.STATUS_VETOED, ) ) self._name = name self._path = path self._mimes = mimes self._file_import_options = file_import_options self._tag_import_options = tag_import_options self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options self._actions = actions self._action_locations = action_locations self._period = period self._check_regularly = check_regularly self._paused = paused self._check_now = check_now self._show_working_popup = show_working_popup self._publish_files_to_popup_button = publish_files_to_popup_button self._publish_files_to_page = publish_files_to_page
def __init__(self): HydrusSerialisable.SerialisableBase.__init__(self) self._gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog() self._file_seed_cache = ClientImportFileSeeds.FileSeedCache() self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud') self._tag_import_options = ClientImportOptions.TagImportOptions( is_default=True) self._paused = False self._downloader_key = HydrusData.GenerateKey() self._lock = threading.Lock() self._files_network_job = None self._gallery_network_job = None self._files_repeating_job = None self._gallery_repeating_job = None HG.client_controller.sub(self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated') HG.client_controller.sub(self, 'NotifyGallerySeedsUpdated', 'gallery_seed_log_gallery_seeds_updated')
def __init__(self): HydrusSerialisable.SerialisableBase.__init__(self) file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud') self._pending_jobs = [] self._gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog() self._file_seed_cache = ClientImportFileSeeds.FileSeedCache() self._file_import_options = file_import_options self._formula_name = 'all files linked by images in page' self._queue_paused = False self._files_paused = False self._parser_status = '' self._current_action = '' self._lock = threading.Lock() self._files_network_job = None self._page_network_job = None self._files_repeating_job = None self._queue_repeating_job = None HG.client_controller.sub(self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated')
def ConvertAllParseResultsToFileSeeds(all_parse_results, source_url, file_import_options): file_seeds = [] seen_urls = set() for parse_results in all_parse_results: parsed_urls = ClientParsing.GetURLsFromParseResults( parse_results, (HC.URL_TYPE_DESIRED, ), only_get_top_priority=True) parsed_urls = HydrusData.DedupeList(parsed_urls) parsed_urls = [url for url in parsed_urls if url not in seen_urls] seen_urls.update(parsed_urls) # note we do this recursively due to parse_results being appropriate only for these urls--don't move this out again, or tags will be messed up for url in parsed_urls: file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.SetReferralURL(source_url) file_seed.AddParseResults(parse_results, file_import_options) file_seeds.append(file_seed) return file_seeds
def _RegenerateStatus( self ): file_seed_caches = [ watcher.GetFileSeedCache() for watcher in self._watchers ] self._status_cache = ClientImportFileSeeds.GenerateFileSeedCachesStatus( file_seed_caches ) self._status_dirty = False self._status_cache_generation_time = HydrusData.GetNow()
def THREADDownloadURL(job_key, url, url_string): job_key.SetVariable('popup_title', url_string) job_key.SetVariable('popup_text_1', 'downloading and importing') # file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud') def network_job_factory(*args, **kwargs): network_job = ClientNetworkingJobs.NetworkJob(*args, **kwargs) network_job.OverrideBandwidth(30) return network_job network_job_presentation_context_factory = GenerateSinglePopupNetworkJobPresentationContextFactory( job_key) file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) # try: file_seed.DownloadAndImportRawFile( url, file_import_options, network_job_factory, network_job_presentation_context_factory) status = file_seed.status if status in CC.SUCCESSFUL_IMPORT_STATES: if status == CC.STATUS_SUCCESSFUL_AND_NEW: job_key.SetVariable('popup_text_1', 'successful!') elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT: job_key.SetVariable('popup_text_1', 'was already in the database!') if file_seed.HasHash(): hash = file_seed.GetHash() job_key.SetVariable('popup_files', ([hash], 'download')) elif status == CC.STATUS_DELETED: job_key.SetVariable('popup_text_1', 'had already been deleted!') finally: job_key.Finish()
def __init__( self, name, path = '', file_import_options = None, tag_import_options = None, tag_service_keys_to_filename_tagging_options = None, mimes = None, actions = None, action_locations = None, period = 3600, check_regularly = True, show_working_popup = True, publish_files_to_popup_button = True, publish_files_to_page = False ): if mimes is None: mimes = HC.ALLOWED_MIMES if file_import_options is None: file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'quiet' ) if tag_import_options is None: tag_import_options = ClientImportOptions.TagImportOptions() if tag_service_keys_to_filename_tagging_options is None: tag_service_keys_to_filename_tagging_options = {} if actions is None: actions = {} actions[ CC.STATUS_SUCCESSFUL_AND_NEW ] = CC.IMPORT_FOLDER_IGNORE actions[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ] = CC.IMPORT_FOLDER_IGNORE actions[ CC.STATUS_DELETED ] = CC.IMPORT_FOLDER_IGNORE actions[ CC.STATUS_ERROR ] = CC.IMPORT_FOLDER_IGNORE if action_locations is None: action_locations = {} HydrusSerialisable.SerialisableBaseNamed.__init__( self, name ) self._path = path self._mimes = mimes self._file_import_options = file_import_options self._tag_import_options = tag_import_options self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options self._actions = actions self._action_locations = action_locations self._period = period self._check_regularly = check_regularly self._file_seed_cache = ClientImportFileSeeds.FileSeedCache() self._last_checked = 0 self._paused = False self._check_now = False self._show_working_popup = show_working_popup self._publish_files_to_popup_button = publish_files_to_popup_button self._publish_files_to_page = publish_files_to_page
def __init__(self): HydrusSerialisable.SerialisableBase.__init__(self) self._page_key = 'initialising page key' self._publish_to_page = False self._url = '' self._gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog() self._file_seed_cache = ClientImportFileSeeds.FileSeedCache() self._urls_to_filenames = {} self._urls_to_md5_base64 = {} self._checker_options = HG.client_controller.new_options.GetDefaultWatcherCheckerOptions( ) self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud') self._tag_import_options = ClientImportOptions.TagImportOptions( is_default=True) self._last_check_time = 0 self._checking_status = ClientImporting.CHECKER_STATUS_OK self._subject = 'unknown subject' self._next_check_time = None self._download_control_file_set = None self._download_control_file_clear = None self._download_control_checker_set = None self._download_control_checker_clear = None self._check_now = False self._files_paused = False self._checking_paused = False self._no_work_until = 0 self._no_work_until_reason = '' self._creation_time = HydrusData.GetNow() self._file_velocity_status = '' self._current_action = '' self._watcher_status = '' self._watcher_key = HydrusData.GenerateKey() self._lock = threading.Lock() self._last_pubbed_page_name = '' self._files_repeating_job = None self._checker_repeating_job = None HG.client_controller.sub(self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated')
def _RegenerateStatus(self): file_seed_caches = [ gallery_import.GetFileSeedCache() for gallery_import in self._gallery_imports ] self._status_cache = ClientImportFileSeeds.GenerateFileSeedCachesStatus( file_seed_caches) self._status_dirty = False self._status_cache_generation_time = HydrusData.GetNow()
def UpdateFileSeedCacheWithAllParseResults(file_seed_cache, all_parse_results, source_url, max_new_urls_allowed=None): new_file_seeds = [] num_urls_added = 0 num_urls_already_in_file_seed_cache = 0 num_urls_total = 0 for parse_results in all_parse_results: parsed_urls = ClientParsing.GetURLsFromParseResults( parse_results, (HC.URL_TYPE_DESIRED, ), only_get_top_priority=True) for url in parsed_urls: num_urls_total += 1 if max_new_urls_allowed is not None and num_urls_added == max_new_urls_allowed: continue file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.SetReferralURL(source_url) if file_seed_cache.HasFileSeed(file_seed): num_urls_already_in_file_seed_cache += 1 else: num_urls_added += 1 file_seed.AddParseResults(parse_results) new_file_seeds.append(file_seed) file_seed_cache.AddFileSeeds(new_file_seeds) return (num_urls_added, num_urls_already_in_file_seed_cache, num_urls_total)
def _ImportSources(self, sources): file_seed_cache = self._file_seed_cache_get_callable() if sources[0].startswith('http'): file_seed_type = ClientImportFileSeeds.FILE_SEED_TYPE_URL else: file_seed_type = ClientImportFileSeeds.FILE_SEED_TYPE_HDD file_seeds = [ ClientImportFileSeeds.FileSeed(file_seed_type, source) for source in sources ] file_seed_cache.AddFileSeeds(file_seeds)
def PendURLs(self, urls): with self._lock: urls = filter(lambda u: len(u) > 1, urls) # > _1_ to take out the occasional whitespace file_seeds = [] gallery_seeds = [] for url in urls: url_match = HG.client_controller.network_engine.domain_manager.GetURLMatch( url) if url_match is None or url_match.GetURLType() in ( HC.URL_TYPE_FILE, HC.URL_TYPE_POST): file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seeds.append(file_seed) else: can_generate_more_pages = False gallery_seed = ClientImportGallerySeeds.GallerySeed( url, can_generate_more_pages=can_generate_more_pages) gallery_seeds.append(gallery_seed) if len(gallery_seeds) > 0: self._gallery_seed_log.AddGallerySeeds(gallery_seeds) ClientImporting.WakeRepeatingJob(self._gallery_repeating_job) if len(file_seeds) > 0: self._file_seed_cache.AddFileSeeds(file_seeds) ClientImporting.WakeRepeatingJob(self._files_repeating_job)
def GetPage( self, gallery_url ): data = self._FetchData( gallery_url ) ( page_of_urls_and_tags, definitely_no_more_pages ) = self._ParseGalleryPage( data, gallery_url ) import ClientImportFileSeeds page_of_file_seeds = [] for ( url, tags ) in page_of_urls_and_tags: file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url ) file_seed.AddTags( tags ) page_of_file_seeds.append( file_seed ) return ( page_of_file_seeds, definitely_no_more_pages )
def _CheckFolder(self, job_key): filenames = os.listdir(HydrusData.ToUnicode(self._path)) raw_paths = [ os.path.join(self._path, filename) for filename in filenames ] all_paths = ClientFiles.GetAllPaths(raw_paths) all_paths = HydrusPaths.FilterFreePaths(all_paths) file_seeds = [] for path in all_paths: if job_key.IsCancelled(): break if path.endswith('.txt'): continue file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path) if not self._file_seed_cache.HasFileSeed(file_seed): file_seeds.append(file_seed) job_key.SetVariable( 'popup_text_1', 'checking: found ' + HydrusData.ToHumanInt(len(file_seeds)) + ' new files') self._file_seed_cache.AddFileSeeds(file_seeds) self._last_checked = HydrusData.GetNow() self._check_now = False
def _WorkOnQueue(self, page_key): if len(self._pending_jobs) > 0: with self._lock: (url, simple_downloader_formula) = self._pending_jobs.pop(0) self._parser_status = 'checking ' + url error_occurred = False try: gallery_seed = ClientImportGallerySeeds.GallerySeed( url, can_generate_more_pages=False) self._gallery_seed_log.AddGallerySeeds((gallery_seed, )) network_job = ClientNetworkingJobs.NetworkJobDownloader( page_key, 'GET', url) network_job.OverrideBandwidth(30) HG.client_controller.network_engine.AddJob(network_job) with self._PageNetworkJobPresentationContextFactory( network_job): network_job.WaitUntilDone() data = network_job.GetContent() # parsing_context = {} parsing_context['url'] = url parsing_formula = simple_downloader_formula.GetFormula() file_seeds = [] for parsed_text in parsing_formula.Parse( parsing_context, data): try: file_url = urlparse.urljoin(url, parsed_text) file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, file_url) file_seed.SetReferralURL(url) file_seeds.append(file_seed) except: continue num_new = self._file_seed_cache.AddFileSeeds(file_seeds) if num_new > 0: ClientImporting.WakeRepeatingJob(self._files_repeating_job) parser_status = 'page checked OK with formula "' + simple_downloader_formula.GetName( ) + '" - ' + HydrusData.ToHumanInt(num_new) + ' new urls' num_already_in_file_seed_cache = len(file_seeds) - num_new if num_already_in_file_seed_cache > 0: parser_status += ' (' + HydrusData.ToHumanInt( num_already_in_file_seed_cache) + ' already in queue)' gallery_seed_status = CC.STATUS_SUCCESSFUL_AND_NEW except HydrusExceptions.ShutdownException: gallery_seed_status = CC.STATUS_VETOED parser_status = 'program is shutting down' return except HydrusExceptions.NotFoundException: gallery_seed_status = CC.STATUS_VETOED error_occurred = True parser_status = 'page 404' except Exception as e: gallery_seed_status = CC.STATUS_ERROR error_occurred = True parser_status = HydrusData.ToUnicode(e) finally: gallery_seed_note = parser_status gallery_seed.SetStatus(gallery_seed_status, note=gallery_seed_note) self._gallery_seed_log.NotifyGallerySeedsUpdated( (gallery_seed, )) with self._lock: self._parser_status = parser_status if error_occurred: time.sleep(5) return True else: with self._lock: self._parser_status = '' return False
def THREADDownloadURLs(job_key, urls, title): job_key.SetVariable('popup_title', title) job_key.SetVariable('popup_text_1', 'initialising') num_successful = 0 num_redundant = 0 num_deleted = 0 num_failed = 0 presentation_hashes = [] presentation_hashes_fast = set() file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud') def network_job_factory(*args, **kwargs): network_job = ClientNetworkingJobs.NetworkJob(*args, **kwargs) network_job.OverrideBandwidth() return network_job network_job_presentation_context_factory = GenerateMultiplePopupNetworkJobPresentationContextFactory( job_key) for (i, url) in enumerate(urls): (i_paused, should_quit) = job_key.WaitIfNeeded() if should_quit: break job_key.SetVariable( 'popup_text_1', HydrusData.ConvertValueRangeToPrettyString(i + 1, len(urls))) job_key.SetVariable('popup_gauge_1', (i + 1, len(urls))) file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) try: file_seed.DownloadAndImportRawFile( url, file_import_options, network_job_factory, network_job_presentation_context_factory) status = file_seed.status if status in CC.SUCCESSFUL_IMPORT_STATES: if status == CC.STATUS_SUCCESSFUL_AND_NEW: num_successful += 1 elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT: num_redundant += 1 if file_seed.HasHash(): hash = file_seed.GetHash() if hash not in presentation_hashes_fast: presentation_hashes.append(hash) presentation_hashes_fast.add(hash) elif status == CC.STATUS_DELETED: num_deleted += 1 except Exception as e: num_failed += 1 HydrusData.Print(url + ' failed to import!') HydrusData.PrintException(e) job_key.DeleteVariable('popup_network_job') text_components = [] if num_successful > 0: text_components.append( HydrusData.ToHumanInt(num_successful) + ' successful') if num_redundant > 0: text_components.append( HydrusData.ToHumanInt(num_redundant) + ' already in db') if num_deleted > 0: text_components.append(HydrusData.ToHumanInt(num_deleted) + ' deleted') if num_failed > 0: text_components.append( HydrusData.ToHumanInt(num_failed) + ' failed (errors written to log)') job_key.SetVariable('popup_text_1', ', '.join(text_components)) if len(presentation_hashes) > 0: job_key.SetVariable('popup_files', (presentation_hashes, 'downloads')) job_key.DeleteVariable('popup_gauge_1') job_key.Finish()
def DAEMONDownloadFiles( controller ): hashes = controller.Read( 'downloads' ) num_downloads = len( hashes ) if num_downloads > 0: client_files_manager = controller.client_files_manager successful_hashes = set() job_key = ClientThreading.JobKey() job_key.SetVariable( 'popup_text_1', 'initialising downloader' ) controller.pub( 'message', job_key ) for hash in hashes: job_key.SetVariable( 'popup_text_1', 'downloading ' + HydrusData.ToHumanInt( num_downloads - len( successful_hashes ) ) + ' files from repositories' ) ( media_result, ) = controller.Read( 'media_results', ( hash, ) ) service_keys = list( media_result.GetLocationsManager().GetCurrent() ) random.shuffle( service_keys ) for service_key in service_keys: if service_key == CC.LOCAL_FILE_SERVICE_KEY: break elif service_key == CC.TRASH_SERVICE_KEY: continue try: service = controller.services_manager.GetService( service_key ) except: continue if service.GetServiceType() == HC.FILE_REPOSITORY: file_repository = service if file_repository.IsFunctional(): try: ( os_file_handle, temp_path ) = ClientPaths.GetTempPath() try: file_repository.Request( HC.GET, 'file', { 'hash' : hash }, temp_path = temp_path ) controller.WaitUntilModelFree() exclude_deleted = False # this is the important part here allow_decompression_bombs = True min_size = None max_size = None max_gif_size = None min_resolution = None max_resolution = None automatic_archive = False file_import_options = ClientImportOptions.FileImportOptions() file_import_options.SetPreImportOptions( exclude_deleted, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution ) file_import_options.SetPostImportOptions( automatic_archive ) file_import_job = ClientImportFileSeeds.FileImportJob( temp_path, file_import_options ) client_files_manager.ImportFile( file_import_job ) successful_hashes.add( hash ) break finally: HydrusPaths.CleanUpTempPath( os_file_handle, temp_path ) except HydrusExceptions.ServerBusyException: job_key.SetVariable( 'popup_text_1', file_repository.GetName() + ' was busy. waiting 30s before trying again' ) time.sleep( 30 ) job_key.Delete() controller.pub( 'notify_new_downloads' ) return except Exception as e: HydrusData.ShowText( 'Error downloading file!' ) HydrusData.ShowException( e ) elif service.GetServiceType() == HC.IPFS: multihashes = HG.client_controller.Read( 'service_filenames', service_key, { hash } ) if len( multihashes ) > 0: multihash = multihashes[0] # this actually calls to a thread that can launch gui 'select from tree' stuff, so let's just break at this point service.ImportFile( multihash ) break if HydrusThreading.IsThreadShuttingDown(): return if len( successful_hashes ) > 0: job_key.SetVariable( 'popup_text_1', HydrusData.ToHumanInt( len( successful_hashes ) ) + ' files downloaded' ) job_key.Delete()
def test_checker_options(self): regular_checker_options = ClientImportOptions.CheckerOptions( intended_files_per_check=5, never_faster_than=30, never_slower_than=86400, death_file_velocity=(1, 86400)) fast_checker_options = ClientImportOptions.CheckerOptions( intended_files_per_check=2, never_faster_than=30, never_slower_than=86400, death_file_velocity=(1, 86400)) slow_checker_options = ClientImportOptions.CheckerOptions( intended_files_per_check=10, never_faster_than=30, never_slower_than=86400, death_file_velocity=(1, 86400)) callous_checker_options = ClientImportOptions.CheckerOptions( intended_files_per_check=5, never_faster_than=30, never_slower_than=86400, death_file_velocity=(1, 60)) empty_file_seed_cache = ClientImportFileSeeds.FileSeedCache() file_seed_cache = ClientImportFileSeeds.FileSeedCache() last_check_time = 10000000 one_day_before = last_check_time - 86400 for i in range(50): url = 'https://wew.lad/' + os.urandom(16).encode('hex') file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.source_time = one_day_before - 10 file_seed_cache.AddFileSeeds((file_seed, )) for i in range(50): url = 'https://wew.lad/' + os.urandom(16).encode('hex') file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.source_time = last_check_time - 600 file_seed_cache.AddFileSeeds((file_seed, )) bare_file_seed_cache = ClientImportFileSeeds.FileSeedCache() url = 'https://wew.lad/' + 'early' file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.source_time = one_day_before - 10 bare_file_seed_cache.AddFileSeeds((file_seed, )) url = 'https://wew.lad/' + 'in_time_delta' file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.source_time = one_day_before + 10 bare_file_seed_cache.AddFileSeeds((file_seed, )) busy_file_seed_cache = ClientImportFileSeeds.FileSeedCache() url = 'https://wew.lad/' + 'early' file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.source_time = one_day_before - 10 busy_file_seed_cache.AddFileSeeds((file_seed, )) for i in range(8640): url = 'https://wew.lad/' + os.urandom(16).encode('hex') file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.source_time = one_day_before + ((i + 1) * 10) - 1 busy_file_seed_cache.AddFileSeeds((file_seed, )) new_thread_file_seed_cache = ClientImportFileSeeds.FileSeedCache() for i in range(10): url = 'https://wew.lad/' + os.urandom(16).encode('hex') file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_URL, url) file_seed.source_time = last_check_time - 600 new_thread_file_seed_cache.AddFileSeeds((file_seed, )) # empty # should say ok if last_check_time is 0, so it can initialise # otherwise sperg out safely self.assertFalse( regular_checker_options.IsDead(empty_file_seed_cache, 0)) self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( empty_file_seed_cache, 0), 'no files yet') self.assertEqual( regular_checker_options.GetNextCheckTime(empty_file_seed_cache, 0, 0), 0) self.assertTrue( regular_checker_options.IsDead(empty_file_seed_cache, last_check_time)) self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( empty_file_seed_cache, last_check_time), 'no files, unable to determine velocity') # regular # current velocity should be 50 files per day for the day ones and 0 files per min for the callous minute one self.assertFalse( regular_checker_options.IsDead(file_seed_cache, last_check_time)) self.assertFalse( fast_checker_options.IsDead(file_seed_cache, last_check_time)) self.assertFalse( slow_checker_options.IsDead(file_seed_cache, last_check_time)) self.assertTrue( callous_checker_options.IsDead(file_seed_cache, last_check_time)) self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( file_seed_cache, last_check_time), u'at last check, found 50 files in previous 1 day') self.assertEqual( fast_checker_options.GetPrettyCurrentVelocity( file_seed_cache, last_check_time), u'at last check, found 50 files in previous 1 day') self.assertEqual( slow_checker_options.GetPrettyCurrentVelocity( file_seed_cache, last_check_time), u'at last check, found 50 files in previous 1 day') self.assertEqual( callous_checker_options.GetPrettyCurrentVelocity( file_seed_cache, last_check_time), u'at last check, found 0 files in previous 1 minute') self.assertEqual( regular_checker_options.GetNextCheckTime(file_seed_cache, last_check_time, 0), last_check_time + 8640) self.assertEqual( fast_checker_options.GetNextCheckTime(file_seed_cache, last_check_time, 0), last_check_time + 3456) self.assertEqual( slow_checker_options.GetNextCheckTime(file_seed_cache, last_check_time, 0), last_check_time + 17280) # bare # 1 files per day self.assertFalse( regular_checker_options.IsDead(bare_file_seed_cache, last_check_time)) self.assertTrue( callous_checker_options.IsDead(bare_file_seed_cache, last_check_time)) self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( bare_file_seed_cache, last_check_time), u'at last check, found 1 files in previous 1 day') self.assertEqual( regular_checker_options.GetNextCheckTime(bare_file_seed_cache, last_check_time, 0), last_check_time + 86400) self.assertEqual( fast_checker_options.GetNextCheckTime(bare_file_seed_cache, last_check_time, 0), last_check_time + 86400) self.assertEqual( slow_checker_options.GetNextCheckTime(bare_file_seed_cache, last_check_time, 0), last_check_time + 86400) # busy # 8640 files per day, 6 files per minute self.assertFalse( regular_checker_options.IsDead(busy_file_seed_cache, last_check_time)) self.assertFalse( fast_checker_options.IsDead(busy_file_seed_cache, last_check_time)) self.assertFalse( slow_checker_options.IsDead(busy_file_seed_cache, last_check_time)) self.assertFalse( callous_checker_options.IsDead(busy_file_seed_cache, last_check_time)) self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( busy_file_seed_cache, last_check_time), u'at last check, found 8,640 files in previous 1 day') self.assertEqual( callous_checker_options.GetPrettyCurrentVelocity( busy_file_seed_cache, last_check_time), u'at last check, found 6 files in previous 1 minute') self.assertEqual( regular_checker_options.GetNextCheckTime(busy_file_seed_cache, last_check_time, 0), last_check_time + 50) self.assertEqual( fast_checker_options.GetNextCheckTime(busy_file_seed_cache, last_check_time, 0), last_check_time + 30) self.assertEqual( slow_checker_options.GetNextCheckTime(busy_file_seed_cache, last_check_time, 0), last_check_time + 100) self.assertEqual( callous_checker_options.GetNextCheckTime(busy_file_seed_cache, last_check_time, 0), last_check_time + 50) # new thread # only had files from ten mins ago, so timings are different self.assertFalse( regular_checker_options.IsDead(new_thread_file_seed_cache, last_check_time)) self.assertFalse( fast_checker_options.IsDead(new_thread_file_seed_cache, last_check_time)) self.assertFalse( slow_checker_options.IsDead(new_thread_file_seed_cache, last_check_time)) self.assertTrue( callous_checker_options.IsDead(new_thread_file_seed_cache, last_check_time)) self.assertEqual( regular_checker_options.GetPrettyCurrentVelocity( new_thread_file_seed_cache, last_check_time), u'at last check, found 10 files in previous 10 minutes') self.assertEqual( fast_checker_options.GetPrettyCurrentVelocity( new_thread_file_seed_cache, last_check_time), u'at last check, found 10 files in previous 10 minutes') self.assertEqual( slow_checker_options.GetPrettyCurrentVelocity( new_thread_file_seed_cache, last_check_time), u'at last check, found 10 files in previous 10 minutes') self.assertEqual( callous_checker_options.GetPrettyCurrentVelocity( new_thread_file_seed_cache, last_check_time), u'at last check, found 0 files in previous 1 minute') # these would be 360, 120, 600, but the 'don't check faster the time since last file post' bumps this up self.assertEqual( regular_checker_options.GetNextCheckTime( new_thread_file_seed_cache, last_check_time, 0), last_check_time + 600) self.assertEqual( fast_checker_options.GetNextCheckTime(new_thread_file_seed_cache, last_check_time, 0), last_check_time + 600) self.assertEqual( slow_checker_options.GetNextCheckTime(new_thread_file_seed_cache, last_check_time, 0), last_check_time + 600) # Let's test these new static timings, where if faster_than == slower_than, we just add that period to the 'last_next_check_time' (e.g. checking every sunday night) static_checker_options = ClientImportOptions.CheckerOptions( intended_files_per_check=5, never_faster_than=3600, never_slower_than=3600, death_file_velocity=(1, 3600)) self.assertTrue( static_checker_options.IsDead(bare_file_seed_cache, last_check_time)) last_next_check_time = last_check_time - 200 with patch.object(HydrusData, 'GetNow', return_value=last_check_time + 10): self.assertEqual( static_checker_options.GetNextCheckTime( new_thread_file_seed_cache, last_check_time, last_next_check_time), last_next_check_time + 3600)
def __init__(self, query=None, source_name=None, initial_search_urls=None, start_file_queue_paused=False, start_gallery_queue_paused=False): if query is None: query = 'samus_aran' if source_name is None: source_name = 'unknown' if initial_search_urls is None: initial_search_urls = [] HydrusSerialisable.SerialisableBase.__init__(self) self._creation_time = HydrusData.GetNow() self._gallery_import_key = HydrusData.GenerateKey() self._query = query self._source_name = source_name self._page_key = 'initialising page key' self._publish_to_page = False self._current_page_index = 0 self._num_new_urls_found = 0 self._num_urls_found = 0 self._file_limit = HC.options['gallery_file_limit'] self._files_paused = start_file_queue_paused self._gallery_paused = start_gallery_queue_paused self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud') self._tag_import_options = ClientImportOptions.TagImportOptions( is_default=True) self._gallery_seed_log = ClientImportGallerySeeds.GallerySeedLog() gallery_seeds = [ ClientImportGallerySeeds.GallerySeed(url) for url in initial_search_urls ] self._gallery_seed_log.AddGallerySeeds(gallery_seeds) self._file_seed_cache = ClientImportFileSeeds.FileSeedCache() self._no_work_until = 0 self._no_work_until_reason = '' self._lock = threading.Lock() self._gallery_status = '' self._gallery_status_can_change_timestamp = 0 self._current_action = '' self._file_network_job = None self._gallery_network_job = None self._files_repeating_job = None self._gallery_repeating_job = None HG.client_controller.sub(self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated') HG.client_controller.sub(self, 'NotifyGallerySeedsUpdated', 'gallery_seed_log_gallery_seeds_updated')