def _test_file_repo(self, service):
info = service.GetInfo()
info['access_key'] = self._access_key
# file
path = ServerFiles.GetExpectedFilePath(self._file_hash)
HydrusPaths.MakeSureDirectoryExists(os.path.dirname(path))
with open(path, 'wb') as f:
f.write(EXAMPLE_FILE)
response = service.Request(HC.GET, 'file',
{'hash': self._file_hash.encode('hex')})
self.assertEqual(response, EXAMPLE_FILE)
try:
os.remove(path)
except:
pass
path = os.path.join(HC.STATIC_DIR, 'hydrus.png')
with open(path, 'rb') as f:
file = f.read()
service.Request(HC.POST, 'file', {'file': file})
written = HydrusGlobals.test_controller.GetWrite('file')
[(args, kwargs)] = written
(written_service_key, written_account, written_file_dict) = args
self.assertEqual(
written_file_dict['hash'],
'\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
)
self.assertEqual(written_file_dict['ip'], '127.0.0.1')
self.assertEqual(written_file_dict['height'], 200)
self.assertEqual(written_file_dict['width'], 200)
self.assertEqual(written_file_dict['mime'], 2)
self.assertEqual(written_file_dict['size'], 5270)
# ip
(ip, timestamp) = ('94.45.87.123', HydrusData.GetNow() - 100000)
HydrusGlobals.test_controller.SetRead('ip', (ip, timestamp))
response = service.Request(HC.GET, 'ip',
{'hash': self._file_hash.encode('hex')})
self.assertEqual(response['ip'], ip)
self.assertEqual(response['timestamp'], timestamp)
# thumbnail
path = ServerFiles.GetExpectedThumbnailPath(self._file_hash)
HydrusPaths.MakeSureDirectoryExists(os.path.dirname(path))
with open(path, 'wb') as f:
f.write(EXAMPLE_THUMBNAIL)
response = service.Request(HC.GET, 'thumbnail',
{'hash': self._file_hash.encode('hex')})
self.assertEqual(response, EXAMPLE_THUMBNAIL)
try:
os.remove(path)
except:
pass
def _test_repo(self, service):
service_key = service.GetServiceKey()
# num_petitions
num_petitions = [
(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_STATUS_PETITIONED, 23),
(HC.CONTENT_TYPE_TAG_PARENTS, HC.CONTENT_STATUS_PENDING, 0)
]
HydrusGlobals.test_controller.SetRead('num_petitions', num_petitions)
response = service.Request(HC.GET, 'num_petitions')
self.assertEqual(response['num_petitions'], num_petitions)
# petition
action = HC.CONTENT_UPDATE_PETITION
petitioner_account = HydrusNetwork.Account.GenerateUnknownAccount()
reason = 'it sucks'
contents = [
HydrusNetwork.Content(
HC.CONTENT_TYPE_FILES,
[HydrusData.GenerateKey() for i in range(10)])
]
petition = HydrusNetwork.Petition(action, petitioner_account, reason,
contents)
HydrusGlobals.test_controller.SetRead('petition', petition)
response = service.Request(HC.GET, 'petition')
self.assertEqual(response['petition'].GetSerialisableTuple(),
petition.GetSerialisableTuple())
# definitions
definitions_update = HydrusNetwork.DefinitionsUpdate()
if i in range(100, 200):
definitions_update.AddRow(
(HC.DEFINITIONS_TYPE_TAGS, i, 'series:test ' + str(i)))
definitions_update.AddRow((HC.DEFINITIONS_TYPE_HASHES, i + 500,
HydrusData.GenerateKey()))
definitions_update_network_string = definitions_update.DumpToNetworkString(
)
definitions_update_hash = hashlib.sha256(
definitions_update_network_string).digest()
path = ServerFiles.GetExpectedFilePath(definitions_update_hash)
with open(path, 'wb') as f:
f.write(definitions_update_network_string)
response = service.Request(HC.GET, 'update',
{'update_hash': definitions_update_hash})
try:
os.remove(path)
except:
pass
self.assertEqual(response, definitions_update_network_string)
# content
rows = [(random.randint(100, 1000),
[random.randint(100, 1000) for i in range(50)])
for j in range(20)]
content_update = HydrusNetwork.ContentUpdate()
for row in rows:
content_update.AddRow(
(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, row))
content_update_network_string = content_update.DumpToNetworkString()
content_update_hash = hashlib.sha256(
content_update_network_string).digest()
path = ServerFiles.GetExpectedFilePath(content_update_hash)
with open(path, 'wb') as f:
f.write(content_update_network_string)
response = service.Request(HC.GET, 'update',
{'update_hash': content_update_hash})
try:
os.remove(path)
except:
pass
self.assertEqual(response, content_update_network_string)
# metadata
metadata = HydrusNetwork.Metadata()
metadata.AppendUpdate([definitions_update_hash, content_update_hash],
HydrusData.GetNow() - 101000,
HydrusData.GetNow() - 1000,
HydrusData.GetNow() + 100000)
service._metadata = metadata
response = service.Request(HC.GET, 'metadata_slice', {'since': 0})
self.assertEqual(response['metadata_slice'].GetSerialisableTuple(),
metadata.GetSerialisableTuple())
# post content
update = HydrusData.ClientToServerContentUpdatePackage(
{}, hash_ids_to_hashes)
service.Request(HC.POST, 'content_update_package', {'update': update})
written = HydrusGlobals.test_controller.GetWrite('update')
[(args, kwargs)] = written
(written_service_key, written_account, written_update) = args
self.assertEqual(update.GetHashes(), written_update.GetHashes())
def ResetPageChangeTimer(self):
self._timestamps['last_page_change'] = HydrusData.GetNow()
def DoWork(self):
if HydrusData.TimeHasPassed(self._last_checked + self._period):
folder_path = HydrusData.ToUnicode(self._path)
if folder_path != '' and os.path.exists(
folder_path) and os.path.isdir(folder_path):
query_hash_ids = HG.client_controller.Read(
'file_query_ids', self._file_search_context)
media_results = []
i = 0
base = 256
while i < len(query_hash_ids):
if HC.options[
'pause_export_folders_sync'] or HydrusThreading.IsThreadShuttingDown(
):
return
if i == 0: (last_i, i) = (0, base)
else: (last_i, i) = (i, i + base)
sub_query_hash_ids = query_hash_ids[last_i:i]
more_media_results = HG.client_controller.Read(
'media_results_from_ids', sub_query_hash_ids)
media_results.extend(more_media_results)
#
terms = ParseExportPhrase(self._phrase)
previous_filenames = set(os.listdir(folder_path))
sync_filenames = set()
client_files_manager = HG.client_controller.client_files_manager
num_copied = 0
for media_result in media_results:
if HC.options[
'pause_export_folders_sync'] or HydrusThreading.IsThreadShuttingDown(
):
return
hash = media_result.GetHash()
mime = media_result.GetMime()
size = media_result.GetSize()
source_path = client_files_manager.GetFilePath(hash, mime)
filename = GenerateExportFilename(folder_path,
media_result, terms)
dest_path = os.path.join(folder_path, filename)
if filename not in sync_filenames:
copied = HydrusPaths.MirrorFile(source_path, dest_path)
if copied:
num_copied += 1
try:
os.chmod(dest_path,
stat.S_IWRITE | stat.S_IREAD)
except:
pass
sync_filenames.add(filename)
if num_copied > 0:
HydrusData.Print(
'Export folder ' + self._name + ' exported ' +
HydrusData.ConvertIntToPrettyString(num_copied) +
' files.')
if self._export_type == HC.EXPORT_FOLDER_TYPE_SYNCHRONISE:
deletee_filenames = previous_filenames.difference(
sync_filenames)
for deletee_filename in deletee_filenames:
deletee_path = os.path.join(folder_path,
deletee_filename)
ClientData.DeletePath(deletee_path)
if len(deletee_filenames) > 0:
HydrusData.Print('Export folder ' + self._name +
' deleted ' +
HydrusData.ConvertIntToPrettyString(
len(deletee_filenames)) +
' files.')
self._last_checked = HydrusData.GetNow()
HG.client_controller.WriteSynchronous('serialisable', self)
def MaintainDB(self, stop_time=None):
if self.new_options.GetBoolean(
'maintain_similar_files_duplicate_pairs_during_idle'):
phashes_stop_time = stop_time
if phashes_stop_time is None:
phashes_stop_time = HydrusData.GetNow() + 15
self.WriteInterruptable('maintain_similar_files_phashes',
stop_time=phashes_stop_time)
tree_stop_time = stop_time
if tree_stop_time is None:
tree_stop_time = HydrusData.GetNow() + 30
self.WriteInterruptable('maintain_similar_files_tree',
stop_time=tree_stop_time,
abandon_if_other_work_to_do=True)
search_distance = self.new_options.GetInteger(
'similar_files_duplicate_pairs_search_distance')
search_stop_time = stop_time
if search_stop_time is None:
search_stop_time = HydrusData.GetNow() + 60
self.WriteInterruptable('maintain_similar_files_duplicate_pairs',
search_distance,
stop_time=search_stop_time,
abandon_if_other_work_to_do=True)
if stop_time is None or not HydrusData.TimeHasPassed(stop_time):
self.WriteInterruptable('maintain_file_reparsing',
stop_time=stop_time)
if stop_time is None or not HydrusData.TimeHasPassed(stop_time):
self.WriteInterruptable('vacuum', stop_time=stop_time)
if stop_time is None or not HydrusData.TimeHasPassed(stop_time):
self.WriteInterruptable('analyze', stop_time=stop_time)
if stop_time is None or not HydrusData.TimeHasPassed(stop_time):
if HydrusData.TimeHasPassed(
self._timestamps['last_service_info_cache_fatten'] +
(60 * 20)):
self.pub('splash_set_status_text', 'fattening service info')
services = self.services_manager.GetServices()
for service in services:
self.pub('splash_set_status_subtext', service.GetName())
try:
self.Read('service_info', service.GetServiceKey())
except:
pass # sometimes this breaks when a service has just been removed and the client is closing, so ignore the error
self._timestamps[
'last_service_info_cache_fatten'] = HydrusData.GetNow()
def Exit(self):
if HG.emergency_exit:
self.ShutdownView()
self.ShutdownModel()
HydrusData.CleanRunningFile(self.db_dir, 'client')
else:
try:
idle_shutdown_action = self.options['idle_shutdown']
if idle_shutdown_action in (CC.IDLE_ON_SHUTDOWN,
CC.IDLE_ON_SHUTDOWN_ASK_FIRST):
idle_shutdown_max_minutes = self.options[
'idle_shutdown_max_minutes']
time_to_stop = HydrusData.GetNow() + (
idle_shutdown_max_minutes * 60)
if self.ThereIsIdleShutdownWorkDue(time_to_stop):
if idle_shutdown_action == CC.IDLE_ON_SHUTDOWN_ASK_FIRST:
text = 'Is now a good time for the client to do up to ' + HydrusData.ToHumanInt(
idle_shutdown_max_minutes
) + ' minutes\' maintenance work? (Will auto-no in 15 seconds)'
with ClientGUIDialogs.DialogYesNo(
self._splash, text,
title='Maintenance is due') as dlg_yn:
job = self.CallLaterWXSafe(
dlg_yn, 15, dlg_yn.EndModal, wx.ID_NO)
try:
if dlg_yn.ShowModal() == wx.ID_YES:
HG.do_idle_shutdown_work = True
finally:
job.Cancel()
else:
HG.do_idle_shutdown_work = True
self.CallToThreadLongRunning(self.THREADExitEverything)
except:
self._DestroySplash()
HydrusData.DebugPrint(traceback.format_exc())
HG.emergency_exit = True
self.Exit()
def __init__( self, method, url, body = None, referral_url = None, temp_path = None ):
self.engine = None
self._lock = threading.Lock()
self._method = method
self._url = url
self._domain = ClientNetworkingDomain.ConvertURLIntoDomain( self._url )
self._second_level_domain = ClientNetworkingDomain.ConvertURLIntoSecondLevelDomain( self._url )
self._body = body
self._referral_url = referral_url
self._temp_path = temp_path
self._files = None
self._for_login = False
self._current_connection_attempt_number = 1
self._additional_headers = {}
self._creation_time = HydrusData.GetNow()
self._bandwidth_tracker = HydrusNetworking.BandwidthTracker()
self._wake_time = 0
self._content_type = None
self._stream_io = cStringIO.StringIO()
self._error_exception = Exception( 'Exception not initialised.' ) # PyLint hint, wew
self._error_exception = None
self._error_text = None
self._is_done_event = threading.Event()
self._is_started = False
self._is_done = False
self._is_cancelled = False
self._gallery_token_name = None
self._gallery_token_consumed = False
self._bandwidth_manual_override = False
self._bandwidth_manual_override_delayed_timestamp = None
self._last_time_ongoing_bandwidth_failed = 0
self._status_text = u'initialising\u2026'
self._num_bytes_read = 0
self._num_bytes_to_read = 1
self._death_time = None
self._file_import_options = None
self._network_contexts = self._GenerateNetworkContexts()
( self._session_network_context, self._login_network_context ) = self._GenerateSpecificNetworkContexts()
def BandwidthOK( self ):
with self._lock:
if self._ObeysBandwidth():
result = self.engine.bandwidth_manager.TryToStartRequest( self._network_contexts )
if result:
self._bandwidth_tracker.ReportRequestUsed()
else:
bandwidth_waiting_duration = self.engine.bandwidth_manager.GetWaitingEstimate( self._network_contexts )
will_override = self._bandwidth_manual_override_delayed_timestamp is not None
override_coming_first = False
if will_override:
override_waiting_duration = self._bandwidth_manual_override_delayed_timestamp - HydrusData.GetNow()
override_coming_first = override_waiting_duration < bandwidth_waiting_duration
if override_coming_first:
waiting_duration = override_waiting_duration
prefix = 'overriding bandwidth '
waiting_str = HydrusData.TimestampToPrettyTimeDelta( self._bandwidth_manual_override_delayed_timestamp, just_now_string = 'imminently', just_now_threshold = 2 )
else:
waiting_duration = bandwidth_waiting_duration
prefix = 'bandwidth free '
waiting_str = HydrusData.TimestampToPrettyTimeDelta( HydrusData.GetNow() + waiting_duration, just_now_string = 'imminently', just_now_threshold = 2 )
self._status_text = prefix + waiting_str + u'\u2026'
if waiting_duration > 1200:
self._Sleep( 30 )
elif waiting_duration > 120:
self._Sleep( 10 )
elif waiting_duration > 10:
self._Sleep( 1 )
return result
else:
self._bandwidth_tracker.ReportRequestUsed()
self.engine.bandwidth_manager.ReportRequestUsed( self._network_contexts )
return True
def _Sleep( self, seconds ):
self._wake_time = HydrusData.GetNow() + seconds
