def Commit(self): if self._in_transaction: self._c.execute('COMMIT;') self._in_transaction = False self._transaction_contains_writes = False if HG.db_journal_mode == 'WAL' and HydrusData.TimeHasPassed( self._last_wal_checkpoint_time + 1800): self._c.execute('PRAGMA wal_checkpoint(PASSIVE);') self._last_wal_checkpoint_time = HydrusData.GetNow() if HydrusData.TimeHasPassed(self._last_mem_refresh_time + 600): self._c.execute('DETACH mem;') self._c.execute('ATTACH ":memory:" AS mem;') TemporaryIntegerTableNameCache.instance().Clear() self._last_mem_refresh_time = HydrusData.GetNow() else: HydrusData.Print( 'Received a call to commit, but was not in a transaction!')
def _MaintainCache(self): if HydrusData.TimeHasPassed(self._next_cache_maintenance_timestamp): now = HydrusData.GetNow() oldest_second = now - self.MAX_SECONDS_TIME_DELTA oldest_minute = now - self.MAX_MINUTES_TIME_DELTA oldest_hour = now - self.MAX_HOURS_TIME_DELTA oldest_day = now - self.MAX_DAYS_TIME_DELTA def clear_counter(counter, oldest_timestamp): bad_keys = [ timestamp for timestamp in counter.keys() if timestamp < oldest_timestamp ] for bad_key in bad_keys: del counter[bad_key] clear_counter(self._days_bytes, oldest_day) clear_counter(self._days_requests, oldest_day) clear_counter(self._hours_bytes, oldest_hour) clear_counter(self._hours_requests, oldest_hour) clear_counter(self._minutes_bytes, oldest_minute) clear_counter(self._minutes_requests, oldest_minute) clear_counter(self._seconds_bytes, oldest_second) clear_counter(self._seconds_requests, oldest_second) self._next_cache_maintenance_timestamp = HydrusData.GetNow( ) + self.CACHE_MAINTENANCE_TIME_DELTA
def SleepCheck(self): with self._sleep_lock: if HydrusData.TimeHasPassed(self._timestamps['now_awake']): last_sleep_check = self._timestamps['last_sleep_check'] if HydrusData.TimeHasPassed( last_sleep_check + 600 ): # it has been way too long since this method last fired, so we've prob been asleep self._just_woke_from_sleep = True self.ResetIdleTimer( ) # this will stop the background jobs from kicking in as soon as the grace period is over self._timestamps['now_awake'] = HydrusData.GetNow( ) + 15 # enough time for ethernet to get back online and all that else: self._just_woke_from_sleep = False self._timestamps['last_sleep_check'] = HydrusData.GetNow()
def Delete(self, seconds=None): if seconds is None: self._deletion_time = HydrusData.GetNow() else: self._deletion_time = HydrusData.GetNow() + seconds
def __init__(self, c: sqlite3.Cursor, transaction_commit_period: int): self._c = c self._transaction_commit_period = transaction_commit_period self._transaction_start_time = 0 self._in_transaction = False self._transaction_contains_writes = False self._last_mem_refresh_time = HydrusData.GetNow() self._last_wal_checkpoint_time = HydrusData.GetNow()
def Commit(self): if self._in_transaction: self.DoPubSubs() self.CleanPubSubs() self._Execute('COMMIT;') self._in_transaction = False self._transaction_contains_writes = False if HG.db_journal_mode == 'WAL' and HydrusData.TimeHasPassed( self._last_wal_passive_checkpoint_time + WAL_PASSIVE_CHECKPOINT_PERIOD): if HydrusData.TimeHasPassed( self._last_wal_truncate_checkpoint_time + WAL_TRUNCATE_CHECKPOINT_PERIOD): self._Execute('PRAGMA wal_checkpoint(TRUNCATE);') self._last_wal_truncate_checkpoint_time = HydrusData.GetNow( ) else: self._Execute('PRAGMA wal_checkpoint(PASSIVE);') self._last_wal_passive_checkpoint_time = HydrusData.GetNow() if HydrusData.TimeHasPassed(self._last_mem_refresh_time + MEM_REFRESH_PERIOD): self._Execute('DETACH mem;') self._Execute('ATTACH ":memory:" AS mem;') TemporaryIntegerTableNameCache.instance().Clear() self._last_mem_refresh_time = HydrusData.GetNow() if HG.db_journal_mode == 'PERSIST' and HydrusData.TimeHasPassed( self._last_journal_zero_time + JOURNAL_ZERO_PERIOD): self._ZeroJournal() self._last_journal_zero_time = HydrusData.GetNow() else: HydrusData.Print( 'Received a call to commit, but was not in a transaction!')
def __init__( self, db_dir ): HG.controller = self self._name = 'hydrus' self._last_shutdown_was_bad = False self._i_own_running_file = False self.db_dir = db_dir self.db = None pubsub_valid_callable = self._GetPubsubValidCallable() self._pubsub = HydrusPubSub.HydrusPubSub( self, pubsub_valid_callable ) self._daemons = [] self._daemon_jobs = {} self._caches = {} self._managers = {} self._fast_job_scheduler = None self._slow_job_scheduler = None self._thread_slots = {} self._thread_slots[ 'misc' ] = ( 0, 10 ) self._thread_slot_lock = threading.Lock() self._call_to_threads = [] self._long_running_call_to_threads = [] self._thread_pool_busy_status_text = '' self._thread_pool_busy_status_text_new_check_time = 0 self._call_to_thread_lock = threading.Lock() self._timestamps_lock = threading.Lock() self._timestamps = collections.defaultdict( lambda: 0 ) self._timestamps[ 'boot' ] = HydrusData.GetNow() self._timestamps[ 'last_sleep_check' ] = HydrusData.GetNow() self._sleep_lock = threading.Lock() self._just_woke_from_sleep = False self._system_busy = False self._doing_fast_exit = False
def _GetRawUsage(self, bandwidth_type, time_delta): if time_delta is None: dt = self._GetCurrentDateTime() month_time = self._GetMonthTime(dt) if bandwidth_type == HC.BANDWIDTH_TYPE_DATA: return self._months_bytes[month_time] elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS: return self._months_requests[month_time] (window, counter) = self._GetWindowAndCounter(bandwidth_type, time_delta) if time_delta == 1: # the case of 1 poses a problem as our min block width is also 1. we can't have a window of 0.1s to make the transition smooth # if we include the last second's data in an effort to span the whole previous 1000ms, we end up not doing anything until the next second rolls over # this causes 50% consumption as we consume in the second after the one we verified was clear # so, let's just check the current second and be happy with it now = HydrusData.GetNow() if now in counter: return counter[now] else: return 0 else: # we need the 'window' because this tracks brackets from the first timestamp and we want to include if 'since' lands anywhere in the bracket # e.g. if it is 1200 and we want the past 1,000, we also need the bracket starting at 0, which will include 200-999 search_time_delta = time_delta + window now = HydrusData.GetNow() since = now - search_time_delta # we test 'now' as upper bound because a lad once had a motherboard reset and lost his clock time, ending up with a lump of data recorded several decades in the future # I'm pretty sure this ended up in the seconds thing, so all his short-time tests were failing return sum((value for (timestamp, value) in counter.items() if since <= timestamp <= now))
def __init__(self, pausable=False, cancellable=False, maintenance_mode=HC.MAINTENANCE_FORCED, only_start_if_unbusy=False, stop_time=None, cancel_on_shutdown=True): self._key = HydrusData.GenerateKey() self._creation_time = HydrusData.GetNowFloat() self._pausable = pausable self._cancellable = cancellable self._maintenance_mode = maintenance_mode self._only_start_if_unbusy = only_start_if_unbusy self._stop_time = stop_time self._cancel_on_shutdown = cancel_on_shutdown self._start_time = HydrusData.GetNow() self._deleted = threading.Event() self._deletion_time = None self._begun = threading.Event() self._done = threading.Event() self._cancelled = threading.Event() self._paused = threading.Event() self._ui_update_pause_period = 0.1 self._next_ui_update_pause = HydrusData.GetNowFloat( ) + self._ui_update_pause_period self._yield_pause_period = 10 self._next_yield_pause = HydrusData.GetNow() + self._yield_pause_period self._bigger_pause_period = 100 self._next_bigger_pause = HydrusData.GetNow( ) + self._bigger_pause_period self._longer_pause_period = 1000 self._next_longer_pause = HydrusData.GetNow( ) + self._longer_pause_period self._exception = None self._urls = [] self._variable_lock = threading.Lock() self._variables = dict()
def _ShutdownDaemons( self ): for job in self._daemon_jobs.values(): job.Cancel() self._daemon_jobs = {} for daemon in self._daemons: daemon.shutdown() started = HydrusData.GetNow() while True in ( daemon.is_alive() for daemon in self._daemons ): self._ReportShutdownDaemonsStatus() time.sleep( 0.1 ) if HydrusData.TimeHasPassed( started + 30 ): break self._daemons = []
def FetchTags(self): script = self._script_choice.GetValue() if script.UsesUserInput(): message = 'Enter the custom input for the file lookup script.' with ClientGUIDialogs.DialogTextEntry(self, message) as dlg: if dlg.exec() != QW.QDialog.Accepted: return file_identifier = dlg.GetValue() else: (m, ) = self._media file_identifier = script.ConvertMediaToFileIdentifier(m) stop_time = HydrusData.GetNow() + 30 job_key = ClientThreading.JobKey(cancellable=True, stop_time=stop_time) self._script_management.SetJobKey(job_key) self._SetTags([]) HG.client_controller.CallToThread(self.THREADFetchTags, script, job_key, file_identifier)
def GetJSON( self, json_text ): with self._lock: now = HydrusData.GetNow() if json_text not in self._json_to_jsons: json_object = json.loads( json_text ) self._json_to_jsons[ json_text ] = ( now, json_object ) ( last_accessed, json_object ) = self._json_to_jsons[ json_text ] if last_accessed != now: self._json_to_jsons[ json_text ] = ( now, json_object ) if len( self._json_to_jsons ) > 10: self._CleanCache() return json_object
def GetSoup( self, html ): with self._lock: now = HydrusData.GetNow() if html not in self._html_to_soups: soup = ClientParsing.GetSoup( html ) self._html_to_soups[ html ] = ( now, soup ) ( last_accessed, soup ) = self._html_to_soups[ html ] if last_accessed != now: self._html_to_soups[ html ] = ( now, soup ) if len( self._html_to_soups ) > 10: self._CleanCache() return soup
def MainLoop( self ): try: INIT_WAIT = 10 self._wake_event.wait( INIT_WAIT ) while not ( HG.view_shutdown or self._shutdown ): self._controller.WaitUntilViewFree() if self._WorkPermitted() and self._WorkToDo(): try: service_key = self._GetServiceKeyToWorkOn() except HydrusExceptions.NotFoundException: time.sleep( 5 ) continue work_time = self._GetWorkTime( service_key ) still_needs_work = self._controller.WriteSynchronous( 'sync_tag_display_maintenance', service_key, work_time ) self._service_keys_to_needs_work[ service_key ] = still_needs_work wait_time = self._GetAfterWorkWaitTime( service_key ) self._last_loop_work_time = work_time else: wait_time = 10 self._wake_event.wait( wait_time ) self._wake_event.clear() if self._new_data_event.is_set(): time.sleep( 1 ) self._last_last_new_data_event_time = self._last_new_data_event_time self._last_new_data_event_time = HydrusData.GetNow() self._service_keys_to_needs_work = {} self._new_data_event.clear() finally: self._mainloop_finished = True
def GetThreadInfo(thread=None): global NEXT_THREAD_CLEAROUT if HydrusData.TimeHasPassed(NEXT_THREAD_CLEAROUT): ClearOutDeadThreads() NEXT_THREAD_CLEAROUT = HydrusData.GetNow() + 600 if thread is None: thread = threading.current_thread() with THREAD_INFO_LOCK: if thread not in THREADS_TO_THREAD_INFO: thread_info = {} thread_info['shutting_down'] = False THREADS_TO_THREAD_INFO[thread] = thread_info return THREADS_TO_THREAD_INFO[thread]
def RegisterShutdownWork(self): self._Execute('DELETE from last_shutdown_work_time;') self._Execute( 'INSERT INTO last_shutdown_work_time ( last_shutdown_work_time ) VALUES ( ? );', (HydrusData.GetNow(), ))
def CleanUpTempPath(os_file_handle, temp_path): try: os.close(os_file_handle) except OSError: gc.collect() try: os.close(os_file_handle) except OSError: HydrusData.Print('Could not close the temporary file ' + temp_path) return try: os.remove(temp_path) except OSError: with TEMP_PATH_LOCK: IN_USE_TEMP_PATHS.add((HydrusData.GetNow(), temp_path))
def RegisterSyncComplete( self, checker_options: ClientImportOptions.CheckerOptions, query_log_container: SubscriptionQueryLogContainer): self._last_check_time = HydrusData.GetNow() self._check_now = False death_period = checker_options.GetDeathFileVelocityPeriod() compact_before_this_time = self._last_check_time - death_period gallery_seed_log = query_log_container.GetGallerySeedLog() if gallery_seed_log.CanCompact(compact_before_this_time): gallery_seed_log.Compact(compact_before_this_time) file_seed_cache = query_log_container.GetFileSeedCache() if file_seed_cache.CanCompact(compact_before_this_time): file_seed_cache.Compact(compact_before_this_time) self.SyncToQueryLogContainer(checker_options, query_log_container)
def GetWaitingEstimate(self, bandwidth_type, time_delta, max_allowed): with self._lock: if time_delta is None: # this is monthly dt = self._GetCurrentDateTime() (year, month) = (dt.year, dt.month) next_month_year = year if month == 12: next_month_year += 1 next_month = (month % 12) + 1 next_month_dt = datetime.datetime(next_month_year, next_month, 1) next_month_time = int( calendar.timegm(next_month_dt.timetuple())) return HydrusData.GetTimeDeltaUntilTime(next_month_time) else: # we want the highest time_delta at which usage is >= than max_allowed # time_delta subtract that amount is the time we have to wait for usage to be less than max_allowed # e.g. if in the past 24 hours there was a bunch of usage 16 hours ago clogging it up, we'll have to wait ~8 hours (window, counter) = self._GetWindowAndCounter(bandwidth_type, time_delta) time_delta_in_which_bandwidth_counts = time_delta + window time_and_values = list(counter.items()) time_and_values.sort(reverse=True) now = HydrusData.GetNow() usage = 0 for (timestamp, value) in time_and_values: current_search_time_delta = now - timestamp if current_search_time_delta > time_delta_in_which_bandwidth_counts: # we are searching beyond our time delta. no need to wait break usage += value if usage >= max_allowed: return time_delta_in_which_bandwidth_counts - current_search_time_delta return 0
def _GetWeightedApproximateUsage(self, time_delta): SEARCH_DELTA = self.MIN_TIME_DELTA_FOR_USER counter = self._seconds_bytes now = HydrusData.GetNow() since = now - SEARCH_DELTA valid_timestamps = [ timestamp for timestamp in counter.keys() if since <= timestamp <= now ] if len(valid_timestamps) == 0: return 0 # If we want the average speed over past five secs but nothing has happened in sec 4 and 5, we don't want to count them # otherwise your 1MB/s counts as 200KB/s earliest_timestamp = min(valid_timestamps) SAMPLE_DELTA = max(now - earliest_timestamp, 1) total_bytes = sum( (counter[timestamp] for timestamp in valid_timestamps)) time_delta_average_per_sec = total_bytes / SAMPLE_DELTA return time_delta_average_per_sec * time_delta
def __init__(self, url=None, can_generate_more_pages=True): if url is None: url = 'https://nostrils-central.cx/index.php?post=s&tag=hyper_nostrils&page=3' else: try: url = HG.client_controller.network_engine.domain_manager.NormaliseURL( url) except HydrusExceptions.URLClassException: pass HydrusSerialisable.SerialisableBase.__init__(self) self.url = url self._can_generate_more_pages = can_generate_more_pages self._fixed_service_keys_to_tags = ClientTags.ServiceKeysToTags() self.created = HydrusData.GetNow() self.modified = self.created self.status = CC.STATUS_UNKNOWN self.note = '' self._referral_url = None self._force_next_page_url_generation = False
def REPEATINGWorkOnWatchers(self): with self._lock: if ClientImporting.PageImporterShouldStopWorking(self._page_key): self._watchers_repeating_job.Cancel() return if not self._status_dirty: # if we think we are clean for watcher in self._watchers: file_seed_cache = watcher.GetFileSeedCache() if file_seed_cache.GetStatusGenerationTime( ) > self._status_cache_generation_time: # has there has been an update? self._SetDirty() break if HydrusData.TimeHasPassed(self._next_pub_value_check_time): self._next_pub_value_check_time = HydrusData.GetNow() + 5 current_value_range = self.GetValueRange() if current_value_range != self._last_pubbed_value_range: self._last_pubbed_value_range = current_value_range HG.client_controller.pub('refresh_page_name', self._page_key)
def SleepCheck(self): with self._sleep_lock: if HydrusData.TimeHasPassed( self.GetTimestamp('last_sleep_check') + 60 ): # it has been way too long since this method last fired, so we've prob been asleep self._just_woke_from_sleep = True self.ResetIdleTimer( ) # this will stop the background jobs from kicking in as soon as the grace period is over wake_delay_period = self._GetWakeDelayPeriod() self.SetTimestamp( 'now_awake', HydrusData.GetNow() + wake_delay_period ) # enough time for ethernet to get back online and all that self._ShowJustWokeToUser() elif self._just_woke_from_sleep and HydrusData.TimeHasPassed( self.GetTimestamp('now_awake')): self._just_woke_from_sleep = False self.TouchTimestamp('last_sleep_check')
def RenderPageToFile( path, temp_path, page_index ): cmd = [ SWFRENDER_PATH, path, '-o', temp_path, '-p', str( page_index ) ] timeout = HydrusData.GetNow() + 60 sbp_kwargs = HydrusData.GetSubprocessKWArgs() HydrusData.CheckProgramIsNotShuttingDown() p = subprocess.Popen( cmd, **sbp_kwargs ) while p.poll() is None: if HydrusData.TimeHasPassed( timeout ): p.terminate() raise Exception( 'Could not render the swf page within 60 seconds!' ) time.sleep( 0.5 ) HydrusThreading.SubprocessCommunicate( p )
def AnalyzeTable(self, name): do_it = True result = self._Execute( 'SELECT num_rows FROM analyze_timestamps WHERE name = ?;', (name, )).fetchone() if result is not None: (num_rows, ) = result # if we have previously analyzed a table with some data but the table is now empty, we do not want a new analyze if num_rows > 0 and self._TableIsEmpty(name): do_it = False if do_it: self._Execute('ANALYZE ' + name + ';') (num_rows, ) = self._Execute('SELECT COUNT( * ) FROM ' + name + ';').fetchone() self._Execute('DELETE FROM analyze_timestamps WHERE name = ?;', (name, )) self._Execute( 'INSERT OR IGNORE INTO analyze_timestamps ( name, num_rows, timestamp ) VALUES ( ?, ?, ? );', (name, num_rows, HydrusData.GetNow()))
def SimulateWakeFromSleepEvent(self): with self._sleep_lock: self.SetTimestamp('last_sleep_check', HydrusData.GetNow() - 3600) self.SleepCheck()
def GetThreadPoolBusyStatus( self ): if HydrusData.TimeHasPassed( self._thread_pool_busy_status_text_new_check_time ): with self._call_to_thread_lock: num_threads = sum( ( 1 for t in self._call_to_threads if t.CurrentlyWorking() ) ) if num_threads < 4: self._thread_pool_busy_status_text = '' elif num_threads < 10: self._thread_pool_busy_status_text = 'working' elif num_threads < 20: self._thread_pool_busy_status_text = 'busy' else: self._thread_pool_busy_status_text = 'very busy!' self._thread_pool_busy_status_text_new_check_time = HydrusData.GetNow() + 10 return self._thread_pool_busy_status_text
def CheckPermissionToSeeFiles( self, hash_ids ): with self._lock: if self._search_tag_filter.AllowsEverything(): return if self._last_search_results is None: raise HydrusExceptions.BadRequestException( 'It looks like those search results are no longer available--please run the search again!' ) num_files_asked_for = len( hash_ids ) num_files_allowed_to_see = len( self._last_search_results.intersection( hash_ids ) ) if num_files_allowed_to_see != num_files_asked_for: error_text = 'You do not seem to have access to all those files! You asked to see {} files, but you were only authorised to see {} of them!' error_text = error_text.format( HydrusData.ToHumanInt( num_files_asked_for ), HydrusData.ToHumanInt( num_files_allowed_to_see ) ) raise HydrusExceptions.InsufficientCredentialsException( error_text ) self._search_results_timeout = HydrusData.GetNow() + SEARCH_RESULTS_CACHE_TIMEOUT
def _CheckFolder(self, job_key): all_paths = ClientFiles.GetAllFilePaths([self._path]) all_paths = HydrusPaths.FilterFreePaths(all_paths) file_seeds = [] for path in all_paths: if job_key.IsCancelled(): break if path.endswith('.txt'): continue file_seed = ClientImportFileSeeds.FileSeed( ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path) if not self._file_seed_cache.HasFileSeed(file_seed): file_seeds.append(file_seed) job_key.SetVariable( 'popup_text_1', 'checking: found ' + HydrusData.ToHumanInt(len(file_seeds)) + ' new files') self._file_seed_cache.AddFileSeeds(file_seeds) self._last_checked = HydrusData.GetNow() self._check_now = False
def _test_content_creation(self): tag = 'character:samus aran' hash = HydrusData.GenerateKey() mappings_content = HydrusNetwork.Content(HC.CONTENT_TYPE_MAPPINGS, (tag, (hash, ))) mapping_content = HydrusNetwork.Content(HC.CONTENT_TYPE_MAPPING, (tag, hash)) client_to_server_update = HydrusNetwork.ClientToServerUpdate() client_to_server_update.AddContent(HC.CONTENT_UPDATE_PEND, mappings_content) self._write('update', self._tag_service_key, self._tag_service_regular_account, client_to_server_update, HydrusData.GetNow()) # can extend this to generate and fetch an actual update given a timespan # result = self._read('account_from_content', self._tag_service_key, mapping_content) self.assertEqual(result.GetAccountKey(), self._tag_service_regular_account.GetAccountKey())