def GetHashFromPath(path):
h = hashlib.sha256()
with open(path, 'rb') as f:
for block in HC.ReadFileLikeAsBlocks(f, 65536):
h.update(block)
return h.digest()
def _callbackParsePOSTArgs(self, request):
request.content.seek(0)
if not request.requestHeaders.hasHeader('Content-Type'):
raise HydrusExceptions.ForbiddenException(
'No Content-Type header found!')
content_types = request.requestHeaders.getRawHeaders('Content-Type')
content_type = content_types[0]
try:
mime = HC.mime_enum_lookup[content_type]
except:
raise HydrusExceptions.ForbiddenException(
'Did not recognise Content-Type header!')
if mime == HC.APPLICATION_YAML:
yaml_string = request.content.read()
request.hydrus_request_data_usage += len(yaml_string)
hydrus_args = yaml.safe_load(yaml_string)
else:
temp_path = HC.GetTempPath()
with open(temp_path, 'wb') as f:
for block in HC.ReadFileLikeAsBlocks(request.content, 65536):
f.write(block)
request.hydrus_request_data_usage += len(block)
hydrus_args = ParseFileArguments(temp_path)
request.hydrus_args = hydrus_args
return request
def _threadDoGETJob(self, request):
hash = request.hydrus_args['hash']
# don't I need to check that we aren't stealing the file from another service?
path = SC.GetPath('thumbnail', hash)
response_context = HC.ResponseContext(200, path=path)
return response_context
def TIMEREventMaintenance(self, event):
sys.stdout.flush()
sys.stderr.flush()
self._CheckIfJustWokeFromSleep()
self._timestamps['last_maintenance_time'] = HC.GetNow()
if not self._just_woke_from_sleep and self.CurrentlyIdle():
self.MaintainDB()
def _threadDoGETJob(self, request):
registration_key = self._parseAccessKey(request)
access_key = HC.app.Read('access_key', registration_key)
body = yaml.safe_dump({'access_key': access_key})
response_context = HC.ResponseContext(200, body=body)
return response_context
def _CheckIfJustWokeFromSleep(self):
last_maintenance_time = self._timestamps['last_maintenance_time']
if last_maintenance_time == 0: return False
# this tests if we probably just woke up from a sleep
if HC.GetNow() - last_maintenance_time > MAINTENANCE_PERIOD + (5 * 60):
self._just_woke_from_sleep = True
else:
self._just_woke_from_sleep = False
def _threadDoGETJob(self, request):
hash = request.hydrus_args['hash']
(ip, timestamp) = HC.app.Read('ip', self._service_key, hash)
body = yaml.safe_dump({'ip': ip, 'timestamp': timestamp})
response_context = HC.ResponseContext(200, body=body)
return response_context
def test_downloads(self):
result = self._read('downloads')
self.assertEqual(result, set())
#
hash = '\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
service_keys_to_content_updates = {}
service_keys_to_content_updates[HC.LOCAL_FILE_SERVICE_KEY] = (
HC.ContentUpdate(HC.CONTENT_DATA_TYPE_FILES,
HC.CONTENT_UPDATE_PENDING, (hash, )), )
self._write('content_updates', service_keys_to_content_updates)
#
result = self._read('downloads')
self.assertEqual(result, {hash})
#
hash = '\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
service_keys_to_content_updates = {}
service_keys_to_content_updates[HC.LOCAL_FILE_SERVICE_KEY] = (
HC.ContentUpdate(HC.CONTENT_DATA_TYPE_FILES,
HC.CONTENT_UPDATE_RESCIND_PENDING, (hash, )), )
self._write('content_updates', service_keys_to_content_updates)
#
result = self._read('downloads')
self.assertEqual(result, set())
def _RefreshParents(self):
service_keys_to_statuses_to_pairs = HC.app.Read('tag_parents')
# first collapse siblings
sibling_manager = HC.app.GetManager('tag_siblings')
collapsed_service_keys_to_statuses_to_pairs = collections.defaultdict(
HC.default_dict_set)
for (service_key,
statuses_to_pairs) in service_keys_to_statuses_to_pairs.items():
if service_key == HC.COMBINED_TAG_SERVICE_KEY: continue
for (status, pairs) in statuses_to_pairs.items():
pairs = sibling_manager.CollapsePairs(pairs)
collapsed_service_keys_to_statuses_to_pairs[service_key][
status] = pairs
# now collapse current and pending
service_keys_to_pairs_flat = HC.default_dict_set()
for (service_key, statuses_to_pairs
) in collapsed_service_keys_to_statuses_to_pairs.items():
pairs_flat = statuses_to_pairs[HC.CURRENT].union(
statuses_to_pairs[HC.PENDING])
service_keys_to_pairs_flat[service_key] = pairs_flat
# now create the combined tag service
combined_pairs_flat = set()
for pairs_flat in service_keys_to_pairs_flat.values():
combined_pairs_flat.update(pairs_flat)
service_keys_to_pairs_flat[
HC.COMBINED_TAG_SERVICE_KEY] = combined_pairs_flat
#
service_keys_to_simple_children_to_parents = BuildServiceKeysToSimpleChildrenToParents(
service_keys_to_pairs_flat)
self._service_keys_to_children_to_parents = BuildServiceKeysToChildrenToParents(
service_keys_to_simple_children_to_parents)
def __init__(self, location, long_timeout=False):
(self._scheme, self._host, self._port) = location
if long_timeout: self._timeout = 600
else: self._timeout = 30
self.lock = threading.Lock()
self._last_request_time = HC.GetNow()
self._RefreshConnection()
def _threadDoGETJob(self, request):
access_key = self._parseAccessKey(request)
verified = HC.app.Read('verify_access_key', self._service_key,
access_key)
body = yaml.safe_dump({'verified': verified})
response_context = HC.ResponseContext(200, body=body)
return response_context
def _threadDoGETJob(self, request):
access_key = self._parseAccessKey(request)
session_manager = HC.app.GetManager('restricted_services_sessions')
(session_key,
expires) = session_manager.AddSession(self._service_key, access_key)
now = HC.GetNow()
max_age = now - expires
cookies = [('session_key', session_key.encode('hex'), {
'max_age': max_age,
'path': '/'
})]
response_context = HC.ResponseContext(200, cookies=cookies)
return response_context
def GetPrettyInfo(self):
(hash, inbox, size, mime, timestamp, width, height, duration,
num_frames, num_words, tags_manager, locations_manager, local_ratings,
remote_ratings) = self._media_result.ToTuple()
info_string = HC.ConvertIntToBytes(
size) + ' ' + HC.mime_string_lookup[mime]
if width is not None and height is not None:
info_string += ' (' + HC.ConvertIntToPrettyString(
width) + 'x' + HC.ConvertIntToPrettyString(height) + ')'
if duration is not None:
info_string += ', ' + HC.ConvertMillisecondsToPrettyTime(duration)
if num_frames is not None:
info_string += ' (' + HC.ConvertIntToPrettyString(
num_frames) + ' frames)'
if num_words is not None:
info_string += ' (' + HC.ConvertIntToPrettyString(
num_words) + ' words)'
return info_string
def InitGUI(self):
self._managers = {}
self._managers['services'] = CC.ServicesManager()
self._managers[
'hydrus_sessions'] = HydrusSessions.HydrusSessionManagerClient()
self._managers['local_booru'] = CC.LocalBooruCache()
self._managers['tag_censorship'] = HydrusTags.TagCensorshipManager()
self._managers['tag_siblings'] = HydrusTags.TagSiblingsManager()
self._managers['tag_parents'] = HydrusTags.TagParentsManager()
self._managers['undo'] = CC.UndoManager()
self._managers[
'web_sessions'] = HydrusSessions.WebSessionManagerClient()
self._fullscreen_image_cache = CC.RenderedImageCache('fullscreen')
self._preview_image_cache = CC.RenderedImageCache('preview')
self._thumbnail_cache = CC.ThumbnailCache()
CC.GlobalBMPs.STATICInitialise()
self._gui = ClientGUI.FrameGUI()
HC.pubsub.sub(self, 'Clipboard', 'clipboard')
HC.pubsub.sub(self, 'RestartServer', 'restart_server')
HC.pubsub.sub(self, 'RestartBooru', 'restart_booru')
self.Bind(wx.EVT_TIMER,
self.TIMEREventMaintenance,
id=ID_MAINTENANCE_EVENT_TIMER)
self._maintenance_event_timer = wx.Timer(self,
ID_MAINTENANCE_EVENT_TIMER)
self._maintenance_event_timer.Start(MAINTENANCE_PERIOD * 1000,
wx.TIMER_CONTINUOUS)
# this is because of some bug in wx C++ that doesn't add these by default
wx.richtext.RichTextBuffer.AddHandler(
wx.richtext.RichTextHTMLHandler())
wx.richtext.RichTextBuffer.AddHandler(wx.richtext.RichTextXMLHandler())
if HC.is_first_start: wx.CallAfter(self._gui.DoFirstStart)
if HC.is_db_updated:
wx.CallLater(
1, HC.ShowText, 'The client has updated to version ' +
HC.u(HC.SOFTWARE_VERSION) + '!')
self.RestartServer()
self.RestartBooru()
self._db.StartDaemons()
def _threadDoPOSTJob(self, request):
account = request.hydrus_account
account_key = account.GetAccountKey()
update = request.hydrus_args['update']
HC.app.Write('update', self._service_key, account_key, update)
response_context = HC.ResponseContext(200)
return response_context
def _test_server_admin(self, host, port):
# set up init connection
service_identifier = HC.ClientServiceIdentifier(
os.urandom(32), HC.SERVER_ADMIN, 'server admin')
credentials = CC.Credentials(host, port)
connection = CC.ConnectionToService(service_identifier, credentials)
# init
access_key = os.urandom(32)
HC.app.SetRead('init', access_key)
response = connection.Get('init')
self.assertEqual(response['access_key'], access_key)
# set up connection
credentials = CC.Credentials(host, port, self._access_key)
connection = CC.ConnectionToService(service_identifier, credentials)
# backup
response = connection.Post('backup')
# services
services = {'message': 'hello'}
HC.app.SetRead('services', services)
response = connection.Get('services')
self.assertEqual(response['services_info'], services)
edit_log = 'blah'
connection.Post('services', edit_log=edit_log)
written = HC.app.GetWrite('services')
[(args, kwargs)] = written
(written_service_identifier, written_edit_log) = args
self.assertEqual(edit_log, written_edit_log)
def MaintainDB(self):
gc.collect()
now = HC.GetNow()
shutdown_timestamps = self.Read('shutdown_timestamps')
if HC.options['maintenance_vacuum_period'] != 0:
if now - shutdown_timestamps[
CC.SHUTDOWN_TIMESTAMP_VACUUM] > HC.options[
'maintenance_vacuum_period']:
self.Write('vacuum')
if HC.options['maintenance_delete_orphans_period'] != 0:
if now - shutdown_timestamps[
CC.SHUTDOWN_TIMESTAMP_DELETE_ORPHANS] > HC.options[
'maintenance_delete_orphans_period']:
self.Write('delete_orphans')
if now - self._timestamps['last_service_info_cache_fatten'] > 60 * 20:
HC.pubsub.pub('set_splash_text', 'fattening service info')
services = self.GetManager('services').GetServices()
for service in services:
try:
self.Read('service_info', service.GetServiceKey())
except:
pass # sometimes this breaks when a service has just been removed and the client is closing, so ignore the error
self._timestamps['last_service_info_cache_fatten'] = HC.GetNow()
HC.pubsub.pub('clear_closed_pages')
def _threadDoGETJob(self, request):
share_key = request.hydrus_args['share_key']
hash = request.hydrus_args['hash']
local_booru_manager = HC.app.GetManager('local_booru')
local_booru_manager.CheckFileAuthorised(share_key, hash)
path = CC.GetFilePath(hash)
response_context = HC.ResponseContext(200, path=path)
return response_context
def DoHydrusBandwidth(service_key, method, command, size):
try:
service = HC.app.GetManager('services').GetService(service_key)
except:
return
service_type = service.GetServiceType()
if (service_type, method, command) in HC.BANDWIDTH_CONSUMING_REQUESTS:
HC.pubsub.pub('service_updates_delayed', {
service_key:
[HC.ServiceUpdate(HC.SERVICE_UPDATE_REQUEST_MADE, size)]
})
def test_md5_status(self):
self._clear_db()
hash = '\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
md5 = 'fdadb2cae78f2dfeb629449cd005f2a2'.decode('hex')
path = HC.STATIC_DIR + os.path.sep + 'hydrus.png'
#
result = self._read('md5_status', md5)
self.assertEqual(result, ('new', None))
#
self._write('import_file', path)
#
result = self._read('md5_status', md5)
self.assertEqual(result, ('redundant', hash))
#
content_update = HC.ContentUpdate(HC.CONTENT_DATA_TYPE_FILES,
HC.CONTENT_UPDATE_DELETE, (hash, ))
service_keys_to_content_updates = {
HC.LOCAL_FILE_SERVICE_KEY: (content_update, )
}
self._write('content_updates', service_keys_to_content_updates)
#
HC.options['exclude_deleted_files'] = True
result = self._read('md5_status', md5)
self.assertEqual(result, ('deleted', None))
HC.options['exclude_deleted_files'] = False
result = self._read('md5_status', md5)
self.assertEqual(result, ('new', None))
def THREADDoFileQuery(self, query_key, search_context):
try:
query_hash_ids = self.Read('file_query_ids', search_context)
query_hash_ids = list(query_hash_ids)
random.shuffle(query_hash_ids)
limit = search_context.GetSystemPredicates().GetLimit()
if limit is not None: query_hash_ids = query_hash_ids[:limit]
service_key = search_context.GetFileServiceKey()
include_current_tags = search_context.IncludeCurrentTags()
media_results = []
include_pending_tags = search_context.IncludePendingTags()
i = 0
base = 256
while i < len(query_hash_ids):
if query_key.IsCancelled(): return
if i == 0: (last_i, i) = (0, base)
else: (last_i, i) = (i, i + base)
sub_query_hash_ids = query_hash_ids[last_i:i]
more_media_results = self.Read('media_results_from_ids',
service_key, sub_query_hash_ids)
media_results.extend(more_media_results)
HC.pubsub.pub('set_num_query_results', len(media_results),
len(query_hash_ids))
self.WaitUntilGoodTimeToUseGUIThread()
HC.pubsub.pub('file_query_done', query_key, media_results)
except Exception as e:
HC.ShowException(e)
def _RefreshConnection(self):
if self._scheme == 'http':
self._connection = httplib.HTTPConnection(self._host,
self._port,
timeout=self._timeout)
elif self._scheme == 'https':
self._connection = httplib.HTTPSConnection(self._host,
self._port,
timeout=self._timeout)
try:
self._connection.connect()
except:
raise Exception('Could not connect to ' + HC.u(self._host) + '!')
def BuildSimpleChildrenToParents(pairs):
simple_children_to_parents = HC.default_dict_set()
for (child, parent) in pairs:
if child == parent: continue
if LoopInSimpleChildrenToParents(simple_children_to_parents, child,
parent):
continue
simple_children_to_parents[child].add(parent)
return simple_children_to_parents
def test_4chan_pass(self):
result = self._read('4chan_pass')
self.assertTrue(result, ('', '', 0))
token = 'token'
pin = 'pin'
timeout = HC.GetNow() + 100000
self._write('4chan_pass', (token, pin, timeout))
result = self._read('4chan_pass')
self.assertTrue(result, (token, pin, timeout))
def _threadDoPOSTJob(self, request):
account = request.hydrus_account
account_key = account.GetAccountKey()
file_dict = request.hydrus_args
file_dict['ip'] = request.getClientIP()
HC.app.Write('file', self._service_key, account_key, file_dict)
response_context = HC.ResponseContext(200)
return response_context
def StartServer(*args, **kwargs):
try:
connection = httplib.HTTPConnection('127.0.0.1',
port,
timeout=10)
try:
connection.connect()
connection.close()
text = 'The client\'s local server could not start because something was already bound to port ' + HC.u(
port) + '.'
text += os.linesep * 2
text += 'This usually means another hydrus client is already running and occupying that port. It could be a previous instantiation of this client that has yet to shut itself down.'
text += os.linesep * 2
text += 'You can change the port this client tries to host its local server on in file->options.'
wx.CallLater(1, HC.ShowText, text)
except:
self._local_service = reactor.listenTCP(
port,
HydrusServer.HydrusServiceLocal(
HC.LOCAL_FILE_SERVICE_KEY, HC.LOCAL_FILE,
'This is the local file service.'))
connection = httplib.HTTPConnection('127.0.0.1',
port,
timeout=10)
try:
connection.connect()
connection.close()
except:
text = 'Tried to bind port ' + HC.u(
port) + ' for the local server, but it failed.'
wx.CallLater(1, HC.ShowText, text)
except Exception as e:
wx.CallAfter(HC.ShowException, e)
def _threadDoGETJob(self, request):
subject_identifier = request.hydrus_args['subject_identifier']
subject_account_key = HC.app.Read('account_key_from_identifier',
self._service_key,
subject_identifier)
account_info = HC.app.Read('account_info', self._service_key,
subject_account_key)
body = yaml.safe_dump({'account_info': account_info})
response_context = HC.ResponseContext(200, body=body)
return response_context
def setUpClass(self):
threading.Thread(target=reactor.run,
kwargs={
'installSignalHandlers': 0
}).start()
services = []
self._local_service_identifier = HC.ServerServiceIdentifier(
'local file', HC.LOCAL_FILE)
self._file_service_identifier = HC.ServerServiceIdentifier(
'file service', HC.FILE_REPOSITORY)
self._tag_service_identifier = HC.ServerServiceIdentifier(
'tag service', HC.TAG_REPOSITORY)
permissions = [
HC.GET_DATA, HC.POST_DATA, HC.POST_PETITIONS, HC.RESOLVE_PETITIONS,
HC.MANAGE_USERS, HC.GENERAL_ADMIN, HC.EDIT_SERVICES
]
account_id = 1
account_type = HC.AccountType('account', permissions, (None, None))
created = HC.GetNow() - 100000
expires = None
used_data = (0, 0)
self._account = HC.Account(account_id, account_type, created, expires,
used_data)
self._access_key = os.urandom(32)
self._file_hash = os.urandom(32)
def TWISTEDSetup():
reactor.listenTCP(
HC.DEFAULT_SERVER_ADMIN_PORT,
HydrusServer.HydrusServiceAdmin(HC.SERVER_ADMIN_IDENTIFIER,
'hello'))
reactor.listenTCP(
HC.DEFAULT_LOCAL_FILE_PORT,
HydrusServer.HydrusServiceLocal(self._local_service_identifier,
'hello'))
reactor.listenTCP(
HC.DEFAULT_SERVICE_PORT,
HydrusServer.HydrusServiceRepositoryFile(
self._file_service_identifier, 'hello'))
reactor.listenTCP(
HC.DEFAULT_SERVICE_PORT + 1,
HydrusServer.HydrusServiceRepositoryTag(
self._tag_service_identifier, 'hello'))
reactor.callFromThread(TWISTEDSetup)
time.sleep(1)
def GetAccount(self, service_key, session_key):
with self._lock:
service_sessions = self._service_keys_to_sessions[service_key]
if session_key in service_sessions:
(account, expires) = service_sessions[session_key]
now = HC.GetNow()
if now > expires: del service_sessions[session_key]
else: return account
raise HydrusExceptions.SessionException()
def _recordDataUsage(self, request):
p1 = request.method == 'GET' and self.RECORD_GET_DATA_USAGE
p2 = request.method == 'POST' and self.RECORD_POST_DATA_USAGE
if p1 or p2:
num_bytes = request.hydrus_request_data_usage
HC.pubsub.pub(
'service_updates_delayed', {
HC.LOCAL_BOORU_SERVICE_KEY: [
HC.ServiceUpdate(HC.SERVICE_UPDATE_REQUEST_MADE,
num_bytes)
]
})