def Commit(self):
if self._in_transaction:
self._c.execute('COMMIT;')
self._in_transaction = False
self._transaction_contains_writes = False
if HG.db_journal_mode == 'WAL' and HydrusData.TimeHasPassed(
self._last_wal_checkpoint_time + 1800):
self._c.execute('PRAGMA wal_checkpoint(PASSIVE);')
self._last_wal_checkpoint_time = HydrusData.GetNow()
if HydrusData.TimeHasPassed(self._last_mem_refresh_time + 600):
self._c.execute('DETACH mem;')
self._c.execute('ATTACH ":memory:" AS mem;')
TemporaryIntegerTableNameCache.instance().Clear()
self._last_mem_refresh_time = HydrusData.GetNow()
else:
HydrusData.Print(
'Received a call to commit, but was not in a transaction!')
def _MaintainCache(self):
if HydrusData.TimeHasPassed(self._next_cache_maintenance_timestamp):
now = HydrusData.GetNow()
oldest_second = now - self.MAX_SECONDS_TIME_DELTA
oldest_minute = now - self.MAX_MINUTES_TIME_DELTA
oldest_hour = now - self.MAX_HOURS_TIME_DELTA
oldest_day = now - self.MAX_DAYS_TIME_DELTA
def clear_counter(counter, oldest_timestamp):
bad_keys = [
timestamp for timestamp in counter.keys()
if timestamp < oldest_timestamp
]
for bad_key in bad_keys:
del counter[bad_key]
clear_counter(self._days_bytes, oldest_day)
clear_counter(self._days_requests, oldest_day)
clear_counter(self._hours_bytes, oldest_hour)
clear_counter(self._hours_requests, oldest_hour)
clear_counter(self._minutes_bytes, oldest_minute)
clear_counter(self._minutes_requests, oldest_minute)
clear_counter(self._seconds_bytes, oldest_second)
clear_counter(self._seconds_requests, oldest_second)
self._next_cache_maintenance_timestamp = HydrusData.GetNow(
) + self.CACHE_MAINTENANCE_TIME_DELTA
def SleepCheck(self):
with self._sleep_lock:
if HydrusData.TimeHasPassed(self._timestamps['now_awake']):
last_sleep_check = self._timestamps['last_sleep_check']
if HydrusData.TimeHasPassed(
last_sleep_check + 600
): # it has been way too long since this method last fired, so we've prob been asleep
self._just_woke_from_sleep = True
self.ResetIdleTimer(
) # this will stop the background jobs from kicking in as soon as the grace period is over
self._timestamps['now_awake'] = HydrusData.GetNow(
) + 15 # enough time for ethernet to get back online and all that
else:
self._just_woke_from_sleep = False
self._timestamps['last_sleep_check'] = HydrusData.GetNow()
def Delete(self, seconds=None):
if seconds is None:
self._deletion_time = HydrusData.GetNow()
else:
self._deletion_time = HydrusData.GetNow() + seconds
def __init__(self, c: sqlite3.Cursor, transaction_commit_period: int):
self._c = c
self._transaction_commit_period = transaction_commit_period
self._transaction_start_time = 0
self._in_transaction = False
self._transaction_contains_writes = False
self._last_mem_refresh_time = HydrusData.GetNow()
self._last_wal_checkpoint_time = HydrusData.GetNow()
def Commit(self):
if self._in_transaction:
self.DoPubSubs()
self.CleanPubSubs()
self._Execute('COMMIT;')
self._in_transaction = False
self._transaction_contains_writes = False
if HG.db_journal_mode == 'WAL' and HydrusData.TimeHasPassed(
self._last_wal_passive_checkpoint_time +
WAL_PASSIVE_CHECKPOINT_PERIOD):
if HydrusData.TimeHasPassed(
self._last_wal_truncate_checkpoint_time +
WAL_TRUNCATE_CHECKPOINT_PERIOD):
self._Execute('PRAGMA wal_checkpoint(TRUNCATE);')
self._last_wal_truncate_checkpoint_time = HydrusData.GetNow(
)
else:
self._Execute('PRAGMA wal_checkpoint(PASSIVE);')
self._last_wal_passive_checkpoint_time = HydrusData.GetNow()
if HydrusData.TimeHasPassed(self._last_mem_refresh_time +
MEM_REFRESH_PERIOD):
self._Execute('DETACH mem;')
self._Execute('ATTACH ":memory:" AS mem;')
TemporaryIntegerTableNameCache.instance().Clear()
self._last_mem_refresh_time = HydrusData.GetNow()
if HG.db_journal_mode == 'PERSIST' and HydrusData.TimeHasPassed(
self._last_journal_zero_time + JOURNAL_ZERO_PERIOD):
self._ZeroJournal()
self._last_journal_zero_time = HydrusData.GetNow()
else:
HydrusData.Print(
'Received a call to commit, but was not in a transaction!')
def __init__( self, db_dir ):
HG.controller = self
self._name = 'hydrus'
self._last_shutdown_was_bad = False
self._i_own_running_file = False
self.db_dir = db_dir
self.db = None
pubsub_valid_callable = self._GetPubsubValidCallable()
self._pubsub = HydrusPubSub.HydrusPubSub( self, pubsub_valid_callable )
self._daemons = []
self._daemon_jobs = {}
self._caches = {}
self._managers = {}
self._fast_job_scheduler = None
self._slow_job_scheduler = None
self._thread_slots = {}
self._thread_slots[ 'misc' ] = ( 0, 10 )
self._thread_slot_lock = threading.Lock()
self._call_to_threads = []
self._long_running_call_to_threads = []
self._thread_pool_busy_status_text = ''
self._thread_pool_busy_status_text_new_check_time = 0
self._call_to_thread_lock = threading.Lock()
self._timestamps_lock = threading.Lock()
self._timestamps = collections.defaultdict( lambda: 0 )
self._timestamps[ 'boot' ] = HydrusData.GetNow()
self._timestamps[ 'last_sleep_check' ] = HydrusData.GetNow()
self._sleep_lock = threading.Lock()
self._just_woke_from_sleep = False
self._system_busy = False
self._doing_fast_exit = False
def _GetRawUsage(self, bandwidth_type, time_delta):
if time_delta is None:
dt = self._GetCurrentDateTime()
month_time = self._GetMonthTime(dt)
if bandwidth_type == HC.BANDWIDTH_TYPE_DATA:
return self._months_bytes[month_time]
elif bandwidth_type == HC.BANDWIDTH_TYPE_REQUESTS:
return self._months_requests[month_time]
(window, counter) = self._GetWindowAndCounter(bandwidth_type,
time_delta)
if time_delta == 1:
# the case of 1 poses a problem as our min block width is also 1. we can't have a window of 0.1s to make the transition smooth
# if we include the last second's data in an effort to span the whole previous 1000ms, we end up not doing anything until the next second rolls over
# this causes 50% consumption as we consume in the second after the one we verified was clear
# so, let's just check the current second and be happy with it
now = HydrusData.GetNow()
if now in counter:
return counter[now]
else:
return 0
else:
# we need the 'window' because this tracks brackets from the first timestamp and we want to include if 'since' lands anywhere in the bracket
# e.g. if it is 1200 and we want the past 1,000, we also need the bracket starting at 0, which will include 200-999
search_time_delta = time_delta + window
now = HydrusData.GetNow()
since = now - search_time_delta
# we test 'now' as upper bound because a lad once had a motherboard reset and lost his clock time, ending up with a lump of data recorded several decades in the future
# I'm pretty sure this ended up in the seconds thing, so all his short-time tests were failing
return sum((value for (timestamp, value) in counter.items()
if since <= timestamp <= now))
def __init__(self,
pausable=False,
cancellable=False,
maintenance_mode=HC.MAINTENANCE_FORCED,
only_start_if_unbusy=False,
stop_time=None,
cancel_on_shutdown=True):
self._key = HydrusData.GenerateKey()
self._creation_time = HydrusData.GetNowFloat()
self._pausable = pausable
self._cancellable = cancellable
self._maintenance_mode = maintenance_mode
self._only_start_if_unbusy = only_start_if_unbusy
self._stop_time = stop_time
self._cancel_on_shutdown = cancel_on_shutdown
self._start_time = HydrusData.GetNow()
self._deleted = threading.Event()
self._deletion_time = None
self._begun = threading.Event()
self._done = threading.Event()
self._cancelled = threading.Event()
self._paused = threading.Event()
self._ui_update_pause_period = 0.1
self._next_ui_update_pause = HydrusData.GetNowFloat(
) + self._ui_update_pause_period
self._yield_pause_period = 10
self._next_yield_pause = HydrusData.GetNow() + self._yield_pause_period
self._bigger_pause_period = 100
self._next_bigger_pause = HydrusData.GetNow(
) + self._bigger_pause_period
self._longer_pause_period = 1000
self._next_longer_pause = HydrusData.GetNow(
) + self._longer_pause_period
self._exception = None
self._urls = []
self._variable_lock = threading.Lock()
self._variables = dict()
def _ShutdownDaemons( self ):
for job in self._daemon_jobs.values():
job.Cancel()
self._daemon_jobs = {}
for daemon in self._daemons:
daemon.shutdown()
started = HydrusData.GetNow()
while True in ( daemon.is_alive() for daemon in self._daemons ):
self._ReportShutdownDaemonsStatus()
time.sleep( 0.1 )
if HydrusData.TimeHasPassed( started + 30 ):
break
self._daemons = []
def FetchTags(self):
script = self._script_choice.GetValue()
if script.UsesUserInput():
message = 'Enter the custom input for the file lookup script.'
with ClientGUIDialogs.DialogTextEntry(self, message) as dlg:
if dlg.exec() != QW.QDialog.Accepted:
return
file_identifier = dlg.GetValue()
else:
(m, ) = self._media
file_identifier = script.ConvertMediaToFileIdentifier(m)
stop_time = HydrusData.GetNow() + 30
job_key = ClientThreading.JobKey(cancellable=True, stop_time=stop_time)
self._script_management.SetJobKey(job_key)
self._SetTags([])
HG.client_controller.CallToThread(self.THREADFetchTags, script,
job_key, file_identifier)
def GetJSON( self, json_text ):
with self._lock:
now = HydrusData.GetNow()
if json_text not in self._json_to_jsons:
json_object = json.loads( json_text )
self._json_to_jsons[ json_text ] = ( now, json_object )
( last_accessed, json_object ) = self._json_to_jsons[ json_text ]
if last_accessed != now:
self._json_to_jsons[ json_text ] = ( now, json_object )
if len( self._json_to_jsons ) > 10:
self._CleanCache()
return json_object
def GetSoup( self, html ):
with self._lock:
now = HydrusData.GetNow()
if html not in self._html_to_soups:
soup = ClientParsing.GetSoup( html )
self._html_to_soups[ html ] = ( now, soup )
( last_accessed, soup ) = self._html_to_soups[ html ]
if last_accessed != now:
self._html_to_soups[ html ] = ( now, soup )
if len( self._html_to_soups ) > 10:
self._CleanCache()
return soup
def MainLoop( self ):
try:
INIT_WAIT = 10
self._wake_event.wait( INIT_WAIT )
while not ( HG.view_shutdown or self._shutdown ):
self._controller.WaitUntilViewFree()
if self._WorkPermitted() and self._WorkToDo():
try:
service_key = self._GetServiceKeyToWorkOn()
except HydrusExceptions.NotFoundException:
time.sleep( 5 )
continue
work_time = self._GetWorkTime( service_key )
still_needs_work = self._controller.WriteSynchronous( 'sync_tag_display_maintenance', service_key, work_time )
self._service_keys_to_needs_work[ service_key ] = still_needs_work
wait_time = self._GetAfterWorkWaitTime( service_key )
self._last_loop_work_time = work_time
else:
wait_time = 10
self._wake_event.wait( wait_time )
self._wake_event.clear()
if self._new_data_event.is_set():
time.sleep( 1 )
self._last_last_new_data_event_time = self._last_new_data_event_time
self._last_new_data_event_time = HydrusData.GetNow()
self._service_keys_to_needs_work = {}
self._new_data_event.clear()
finally:
self._mainloop_finished = True
def GetThreadInfo(thread=None):
global NEXT_THREAD_CLEAROUT
if HydrusData.TimeHasPassed(NEXT_THREAD_CLEAROUT):
ClearOutDeadThreads()
NEXT_THREAD_CLEAROUT = HydrusData.GetNow() + 600
if thread is None:
thread = threading.current_thread()
with THREAD_INFO_LOCK:
if thread not in THREADS_TO_THREAD_INFO:
thread_info = {}
thread_info['shutting_down'] = False
THREADS_TO_THREAD_INFO[thread] = thread_info
return THREADS_TO_THREAD_INFO[thread]
def RegisterShutdownWork(self):
self._Execute('DELETE from last_shutdown_work_time;')
self._Execute(
'INSERT INTO last_shutdown_work_time ( last_shutdown_work_time ) VALUES ( ? );',
(HydrusData.GetNow(), ))
def CleanUpTempPath(os_file_handle, temp_path):
try:
os.close(os_file_handle)
except OSError:
gc.collect()
try:
os.close(os_file_handle)
except OSError:
HydrusData.Print('Could not close the temporary file ' + temp_path)
return
try:
os.remove(temp_path)
except OSError:
with TEMP_PATH_LOCK:
IN_USE_TEMP_PATHS.add((HydrusData.GetNow(), temp_path))
def RegisterSyncComplete(
self, checker_options: ClientImportOptions.CheckerOptions,
query_log_container: SubscriptionQueryLogContainer):
self._last_check_time = HydrusData.GetNow()
self._check_now = False
death_period = checker_options.GetDeathFileVelocityPeriod()
compact_before_this_time = self._last_check_time - death_period
gallery_seed_log = query_log_container.GetGallerySeedLog()
if gallery_seed_log.CanCompact(compact_before_this_time):
gallery_seed_log.Compact(compact_before_this_time)
file_seed_cache = query_log_container.GetFileSeedCache()
if file_seed_cache.CanCompact(compact_before_this_time):
file_seed_cache.Compact(compact_before_this_time)
self.SyncToQueryLogContainer(checker_options, query_log_container)
def GetWaitingEstimate(self, bandwidth_type, time_delta, max_allowed):
with self._lock:
if time_delta is None: # this is monthly
dt = self._GetCurrentDateTime()
(year, month) = (dt.year, dt.month)
next_month_year = year
if month == 12:
next_month_year += 1
next_month = (month % 12) + 1
next_month_dt = datetime.datetime(next_month_year, next_month,
1)
next_month_time = int(
calendar.timegm(next_month_dt.timetuple()))
return HydrusData.GetTimeDeltaUntilTime(next_month_time)
else:
# we want the highest time_delta at which usage is >= than max_allowed
# time_delta subtract that amount is the time we have to wait for usage to be less than max_allowed
# e.g. if in the past 24 hours there was a bunch of usage 16 hours ago clogging it up, we'll have to wait ~8 hours
(window,
counter) = self._GetWindowAndCounter(bandwidth_type,
time_delta)
time_delta_in_which_bandwidth_counts = time_delta + window
time_and_values = list(counter.items())
time_and_values.sort(reverse=True)
now = HydrusData.GetNow()
usage = 0
for (timestamp, value) in time_and_values:
current_search_time_delta = now - timestamp
if current_search_time_delta > time_delta_in_which_bandwidth_counts: # we are searching beyond our time delta. no need to wait
break
usage += value
if usage >= max_allowed:
return time_delta_in_which_bandwidth_counts - current_search_time_delta
return 0
def _GetWeightedApproximateUsage(self, time_delta):
SEARCH_DELTA = self.MIN_TIME_DELTA_FOR_USER
counter = self._seconds_bytes
now = HydrusData.GetNow()
since = now - SEARCH_DELTA
valid_timestamps = [
timestamp for timestamp in counter.keys()
if since <= timestamp <= now
]
if len(valid_timestamps) == 0:
return 0
# If we want the average speed over past five secs but nothing has happened in sec 4 and 5, we don't want to count them
# otherwise your 1MB/s counts as 200KB/s
earliest_timestamp = min(valid_timestamps)
SAMPLE_DELTA = max(now - earliest_timestamp, 1)
total_bytes = sum(
(counter[timestamp] for timestamp in valid_timestamps))
time_delta_average_per_sec = total_bytes / SAMPLE_DELTA
return time_delta_average_per_sec * time_delta
def __init__(self, url=None, can_generate_more_pages=True):
if url is None:
url = 'https://nostrils-central.cx/index.php?post=s&tag=hyper_nostrils&page=3'
else:
try:
url = HG.client_controller.network_engine.domain_manager.NormaliseURL(
url)
except HydrusExceptions.URLClassException:
pass
HydrusSerialisable.SerialisableBase.__init__(self)
self.url = url
self._can_generate_more_pages = can_generate_more_pages
self._fixed_service_keys_to_tags = ClientTags.ServiceKeysToTags()
self.created = HydrusData.GetNow()
self.modified = self.created
self.status = CC.STATUS_UNKNOWN
self.note = ''
self._referral_url = None
self._force_next_page_url_generation = False
def REPEATINGWorkOnWatchers(self):
with self._lock:
if ClientImporting.PageImporterShouldStopWorking(self._page_key):
self._watchers_repeating_job.Cancel()
return
if not self._status_dirty: # if we think we are clean
for watcher in self._watchers:
file_seed_cache = watcher.GetFileSeedCache()
if file_seed_cache.GetStatusGenerationTime(
) > self._status_cache_generation_time: # has there has been an update?
self._SetDirty()
break
if HydrusData.TimeHasPassed(self._next_pub_value_check_time):
self._next_pub_value_check_time = HydrusData.GetNow() + 5
current_value_range = self.GetValueRange()
if current_value_range != self._last_pubbed_value_range:
self._last_pubbed_value_range = current_value_range
HG.client_controller.pub('refresh_page_name', self._page_key)
def SleepCheck(self):
with self._sleep_lock:
if HydrusData.TimeHasPassed(
self.GetTimestamp('last_sleep_check') + 60
): # it has been way too long since this method last fired, so we've prob been asleep
self._just_woke_from_sleep = True
self.ResetIdleTimer(
) # this will stop the background jobs from kicking in as soon as the grace period is over
wake_delay_period = self._GetWakeDelayPeriod()
self.SetTimestamp(
'now_awake',
HydrusData.GetNow() + wake_delay_period
) # enough time for ethernet to get back online and all that
self._ShowJustWokeToUser()
elif self._just_woke_from_sleep and HydrusData.TimeHasPassed(
self.GetTimestamp('now_awake')):
self._just_woke_from_sleep = False
self.TouchTimestamp('last_sleep_check')
def RenderPageToFile( path, temp_path, page_index ):
cmd = [ SWFRENDER_PATH, path, '-o', temp_path, '-p', str( page_index ) ]
timeout = HydrusData.GetNow() + 60
sbp_kwargs = HydrusData.GetSubprocessKWArgs()
HydrusData.CheckProgramIsNotShuttingDown()
p = subprocess.Popen( cmd, **sbp_kwargs )
while p.poll() is None:
if HydrusData.TimeHasPassed( timeout ):
p.terminate()
raise Exception( 'Could not render the swf page within 60 seconds!' )
time.sleep( 0.5 )
HydrusThreading.SubprocessCommunicate( p )
def AnalyzeTable(self, name):
do_it = True
result = self._Execute(
'SELECT num_rows FROM analyze_timestamps WHERE name = ?;',
(name, )).fetchone()
if result is not None:
(num_rows, ) = result
# if we have previously analyzed a table with some data but the table is now empty, we do not want a new analyze
if num_rows > 0 and self._TableIsEmpty(name):
do_it = False
if do_it:
self._Execute('ANALYZE ' + name + ';')
(num_rows, ) = self._Execute('SELECT COUNT( * ) FROM ' + name +
';').fetchone()
self._Execute('DELETE FROM analyze_timestamps WHERE name = ?;',
(name, ))
self._Execute(
'INSERT OR IGNORE INTO analyze_timestamps ( name, num_rows, timestamp ) VALUES ( ?, ?, ? );',
(name, num_rows, HydrusData.GetNow()))
def SimulateWakeFromSleepEvent(self):
with self._sleep_lock:
self.SetTimestamp('last_sleep_check', HydrusData.GetNow() - 3600)
self.SleepCheck()
def GetThreadPoolBusyStatus( self ):
if HydrusData.TimeHasPassed( self._thread_pool_busy_status_text_new_check_time ):
with self._call_to_thread_lock:
num_threads = sum( ( 1 for t in self._call_to_threads if t.CurrentlyWorking() ) )
if num_threads < 4:
self._thread_pool_busy_status_text = ''
elif num_threads < 10:
self._thread_pool_busy_status_text = 'working'
elif num_threads < 20:
self._thread_pool_busy_status_text = 'busy'
else:
self._thread_pool_busy_status_text = 'very busy!'
self._thread_pool_busy_status_text_new_check_time = HydrusData.GetNow() + 10
return self._thread_pool_busy_status_text
def CheckPermissionToSeeFiles( self, hash_ids ):
with self._lock:
if self._search_tag_filter.AllowsEverything():
return
if self._last_search_results is None:
raise HydrusExceptions.BadRequestException( 'It looks like those search results are no longer available--please run the search again!' )
num_files_asked_for = len( hash_ids )
num_files_allowed_to_see = len( self._last_search_results.intersection( hash_ids ) )
if num_files_allowed_to_see != num_files_asked_for:
error_text = 'You do not seem to have access to all those files! You asked to see {} files, but you were only authorised to see {} of them!'
error_text = error_text.format( HydrusData.ToHumanInt( num_files_asked_for ), HydrusData.ToHumanInt( num_files_allowed_to_see ) )
raise HydrusExceptions.InsufficientCredentialsException( error_text )
self._search_results_timeout = HydrusData.GetNow() + SEARCH_RESULTS_CACHE_TIMEOUT
def _CheckFolder(self, job_key):
all_paths = ClientFiles.GetAllFilePaths([self._path])
all_paths = HydrusPaths.FilterFreePaths(all_paths)
file_seeds = []
for path in all_paths:
if job_key.IsCancelled():
break
if path.endswith('.txt'):
continue
file_seed = ClientImportFileSeeds.FileSeed(
ClientImportFileSeeds.FILE_SEED_TYPE_HDD, path)
if not self._file_seed_cache.HasFileSeed(file_seed):
file_seeds.append(file_seed)
job_key.SetVariable(
'popup_text_1', 'checking: found ' +
HydrusData.ToHumanInt(len(file_seeds)) + ' new files')
self._file_seed_cache.AddFileSeeds(file_seeds)
self._last_checked = HydrusData.GetNow()
self._check_now = False
def _test_content_creation(self):
tag = 'character:samus aran'
hash = HydrusData.GenerateKey()
mappings_content = HydrusNetwork.Content(HC.CONTENT_TYPE_MAPPINGS,
(tag, (hash, )))
mapping_content = HydrusNetwork.Content(HC.CONTENT_TYPE_MAPPING,
(tag, hash))
client_to_server_update = HydrusNetwork.ClientToServerUpdate()
client_to_server_update.AddContent(HC.CONTENT_UPDATE_PEND,
mappings_content)
self._write('update', self._tag_service_key,
self._tag_service_regular_account, client_to_server_update,
HydrusData.GetNow())
# can extend this to generate and fetch an actual update given a timespan
#
result = self._read('account_from_content', self._tag_service_key,
mapping_content)
self.assertEqual(result.GetAccountKey(),
self._tag_service_regular_account.GetAccountKey())
