def GetHashFromPath(path): h = hashlib.sha256() with open(path, 'rb') as f: for block in HC.ReadFileLikeAsBlocks(f, 65536): h.update(block) return h.digest()
def _callbackParsePOSTArgs(self, request): request.content.seek(0) if not request.requestHeaders.hasHeader('Content-Type'): raise HydrusExceptions.ForbiddenException( 'No Content-Type header found!') content_types = request.requestHeaders.getRawHeaders('Content-Type') content_type = content_types[0] try: mime = HC.mime_enum_lookup[content_type] except: raise HydrusExceptions.ForbiddenException( 'Did not recognise Content-Type header!') if mime == HC.APPLICATION_YAML: yaml_string = request.content.read() request.hydrus_request_data_usage += len(yaml_string) hydrus_args = yaml.safe_load(yaml_string) else: temp_path = HC.GetTempPath() with open(temp_path, 'wb') as f: for block in HC.ReadFileLikeAsBlocks(request.content, 65536): f.write(block) request.hydrus_request_data_usage += len(block) hydrus_args = ParseFileArguments(temp_path) request.hydrus_args = hydrus_args return request
def _threadDoGETJob(self, request): hash = request.hydrus_args['hash'] # don't I need to check that we aren't stealing the file from another service? path = SC.GetPath('thumbnail', hash) response_context = HC.ResponseContext(200, path=path) return response_context
def TIMEREventMaintenance(self, event): sys.stdout.flush() sys.stderr.flush() self._CheckIfJustWokeFromSleep() self._timestamps['last_maintenance_time'] = HC.GetNow() if not self._just_woke_from_sleep and self.CurrentlyIdle(): self.MaintainDB()
def _threadDoGETJob(self, request): registration_key = self._parseAccessKey(request) access_key = HC.app.Read('access_key', registration_key) body = yaml.safe_dump({'access_key': access_key}) response_context = HC.ResponseContext(200, body=body) return response_context
def _CheckIfJustWokeFromSleep(self): last_maintenance_time = self._timestamps['last_maintenance_time'] if last_maintenance_time == 0: return False # this tests if we probably just woke up from a sleep if HC.GetNow() - last_maintenance_time > MAINTENANCE_PERIOD + (5 * 60): self._just_woke_from_sleep = True else: self._just_woke_from_sleep = False
def _threadDoGETJob(self, request): hash = request.hydrus_args['hash'] (ip, timestamp) = HC.app.Read('ip', self._service_key, hash) body = yaml.safe_dump({'ip': ip, 'timestamp': timestamp}) response_context = HC.ResponseContext(200, body=body) return response_context
def test_downloads(self): result = self._read('downloads') self.assertEqual(result, set()) # hash = '\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3' service_keys_to_content_updates = {} service_keys_to_content_updates[HC.LOCAL_FILE_SERVICE_KEY] = ( HC.ContentUpdate(HC.CONTENT_DATA_TYPE_FILES, HC.CONTENT_UPDATE_PENDING, (hash, )), ) self._write('content_updates', service_keys_to_content_updates) # result = self._read('downloads') self.assertEqual(result, {hash}) # hash = '\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3' service_keys_to_content_updates = {} service_keys_to_content_updates[HC.LOCAL_FILE_SERVICE_KEY] = ( HC.ContentUpdate(HC.CONTENT_DATA_TYPE_FILES, HC.CONTENT_UPDATE_RESCIND_PENDING, (hash, )), ) self._write('content_updates', service_keys_to_content_updates) # result = self._read('downloads') self.assertEqual(result, set())
def _RefreshParents(self): service_keys_to_statuses_to_pairs = HC.app.Read('tag_parents') # first collapse siblings sibling_manager = HC.app.GetManager('tag_siblings') collapsed_service_keys_to_statuses_to_pairs = collections.defaultdict( HC.default_dict_set) for (service_key, statuses_to_pairs) in service_keys_to_statuses_to_pairs.items(): if service_key == HC.COMBINED_TAG_SERVICE_KEY: continue for (status, pairs) in statuses_to_pairs.items(): pairs = sibling_manager.CollapsePairs(pairs) collapsed_service_keys_to_statuses_to_pairs[service_key][ status] = pairs # now collapse current and pending service_keys_to_pairs_flat = HC.default_dict_set() for (service_key, statuses_to_pairs ) in collapsed_service_keys_to_statuses_to_pairs.items(): pairs_flat = statuses_to_pairs[HC.CURRENT].union( statuses_to_pairs[HC.PENDING]) service_keys_to_pairs_flat[service_key] = pairs_flat # now create the combined tag service combined_pairs_flat = set() for pairs_flat in service_keys_to_pairs_flat.values(): combined_pairs_flat.update(pairs_flat) service_keys_to_pairs_flat[ HC.COMBINED_TAG_SERVICE_KEY] = combined_pairs_flat # service_keys_to_simple_children_to_parents = BuildServiceKeysToSimpleChildrenToParents( service_keys_to_pairs_flat) self._service_keys_to_children_to_parents = BuildServiceKeysToChildrenToParents( service_keys_to_simple_children_to_parents)
def __init__(self, location, long_timeout=False): (self._scheme, self._host, self._port) = location if long_timeout: self._timeout = 600 else: self._timeout = 30 self.lock = threading.Lock() self._last_request_time = HC.GetNow() self._RefreshConnection()
def _threadDoGETJob(self, request): access_key = self._parseAccessKey(request) verified = HC.app.Read('verify_access_key', self._service_key, access_key) body = yaml.safe_dump({'verified': verified}) response_context = HC.ResponseContext(200, body=body) return response_context
def _threadDoGETJob(self, request): access_key = self._parseAccessKey(request) session_manager = HC.app.GetManager('restricted_services_sessions') (session_key, expires) = session_manager.AddSession(self._service_key, access_key) now = HC.GetNow() max_age = now - expires cookies = [('session_key', session_key.encode('hex'), { 'max_age': max_age, 'path': '/' })] response_context = HC.ResponseContext(200, cookies=cookies) return response_context
def GetPrettyInfo(self): (hash, inbox, size, mime, timestamp, width, height, duration, num_frames, num_words, tags_manager, locations_manager, local_ratings, remote_ratings) = self._media_result.ToTuple() info_string = HC.ConvertIntToBytes( size) + ' ' + HC.mime_string_lookup[mime] if width is not None and height is not None: info_string += ' (' + HC.ConvertIntToPrettyString( width) + 'x' + HC.ConvertIntToPrettyString(height) + ')' if duration is not None: info_string += ', ' + HC.ConvertMillisecondsToPrettyTime(duration) if num_frames is not None: info_string += ' (' + HC.ConvertIntToPrettyString( num_frames) + ' frames)' if num_words is not None: info_string += ' (' + HC.ConvertIntToPrettyString( num_words) + ' words)' return info_string
def InitGUI(self): self._managers = {} self._managers['services'] = CC.ServicesManager() self._managers[ 'hydrus_sessions'] = HydrusSessions.HydrusSessionManagerClient() self._managers['local_booru'] = CC.LocalBooruCache() self._managers['tag_censorship'] = HydrusTags.TagCensorshipManager() self._managers['tag_siblings'] = HydrusTags.TagSiblingsManager() self._managers['tag_parents'] = HydrusTags.TagParentsManager() self._managers['undo'] = CC.UndoManager() self._managers[ 'web_sessions'] = HydrusSessions.WebSessionManagerClient() self._fullscreen_image_cache = CC.RenderedImageCache('fullscreen') self._preview_image_cache = CC.RenderedImageCache('preview') self._thumbnail_cache = CC.ThumbnailCache() CC.GlobalBMPs.STATICInitialise() self._gui = ClientGUI.FrameGUI() HC.pubsub.sub(self, 'Clipboard', 'clipboard') HC.pubsub.sub(self, 'RestartServer', 'restart_server') HC.pubsub.sub(self, 'RestartBooru', 'restart_booru') self.Bind(wx.EVT_TIMER, self.TIMEREventMaintenance, id=ID_MAINTENANCE_EVENT_TIMER) self._maintenance_event_timer = wx.Timer(self, ID_MAINTENANCE_EVENT_TIMER) self._maintenance_event_timer.Start(MAINTENANCE_PERIOD * 1000, wx.TIMER_CONTINUOUS) # this is because of some bug in wx C++ that doesn't add these by default wx.richtext.RichTextBuffer.AddHandler( wx.richtext.RichTextHTMLHandler()) wx.richtext.RichTextBuffer.AddHandler(wx.richtext.RichTextXMLHandler()) if HC.is_first_start: wx.CallAfter(self._gui.DoFirstStart) if HC.is_db_updated: wx.CallLater( 1, HC.ShowText, 'The client has updated to version ' + HC.u(HC.SOFTWARE_VERSION) + '!') self.RestartServer() self.RestartBooru() self._db.StartDaemons()
def _threadDoPOSTJob(self, request): account = request.hydrus_account account_key = account.GetAccountKey() update = request.hydrus_args['update'] HC.app.Write('update', self._service_key, account_key, update) response_context = HC.ResponseContext(200) return response_context
def _test_server_admin(self, host, port): # set up init connection service_identifier = HC.ClientServiceIdentifier( os.urandom(32), HC.SERVER_ADMIN, 'server admin') credentials = CC.Credentials(host, port) connection = CC.ConnectionToService(service_identifier, credentials) # init access_key = os.urandom(32) HC.app.SetRead('init', access_key) response = connection.Get('init') self.assertEqual(response['access_key'], access_key) # set up connection credentials = CC.Credentials(host, port, self._access_key) connection = CC.ConnectionToService(service_identifier, credentials) # backup response = connection.Post('backup') # services services = {'message': 'hello'} HC.app.SetRead('services', services) response = connection.Get('services') self.assertEqual(response['services_info'], services) edit_log = 'blah' connection.Post('services', edit_log=edit_log) written = HC.app.GetWrite('services') [(args, kwargs)] = written (written_service_identifier, written_edit_log) = args self.assertEqual(edit_log, written_edit_log)
def MaintainDB(self): gc.collect() now = HC.GetNow() shutdown_timestamps = self.Read('shutdown_timestamps') if HC.options['maintenance_vacuum_period'] != 0: if now - shutdown_timestamps[ CC.SHUTDOWN_TIMESTAMP_VACUUM] > HC.options[ 'maintenance_vacuum_period']: self.Write('vacuum') if HC.options['maintenance_delete_orphans_period'] != 0: if now - shutdown_timestamps[ CC.SHUTDOWN_TIMESTAMP_DELETE_ORPHANS] > HC.options[ 'maintenance_delete_orphans_period']: self.Write('delete_orphans') if now - self._timestamps['last_service_info_cache_fatten'] > 60 * 20: HC.pubsub.pub('set_splash_text', 'fattening service info') services = self.GetManager('services').GetServices() for service in services: try: self.Read('service_info', service.GetServiceKey()) except: pass # sometimes this breaks when a service has just been removed and the client is closing, so ignore the error self._timestamps['last_service_info_cache_fatten'] = HC.GetNow() HC.pubsub.pub('clear_closed_pages')
def _threadDoGETJob(self, request): share_key = request.hydrus_args['share_key'] hash = request.hydrus_args['hash'] local_booru_manager = HC.app.GetManager('local_booru') local_booru_manager.CheckFileAuthorised(share_key, hash) path = CC.GetFilePath(hash) response_context = HC.ResponseContext(200, path=path) return response_context
def DoHydrusBandwidth(service_key, method, command, size): try: service = HC.app.GetManager('services').GetService(service_key) except: return service_type = service.GetServiceType() if (service_type, method, command) in HC.BANDWIDTH_CONSUMING_REQUESTS: HC.pubsub.pub('service_updates_delayed', { service_key: [HC.ServiceUpdate(HC.SERVICE_UPDATE_REQUEST_MADE, size)] })
def test_md5_status(self): self._clear_db() hash = '\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3' md5 = 'fdadb2cae78f2dfeb629449cd005f2a2'.decode('hex') path = HC.STATIC_DIR + os.path.sep + 'hydrus.png' # result = self._read('md5_status', md5) self.assertEqual(result, ('new', None)) # self._write('import_file', path) # result = self._read('md5_status', md5) self.assertEqual(result, ('redundant', hash)) # content_update = HC.ContentUpdate(HC.CONTENT_DATA_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, (hash, )) service_keys_to_content_updates = { HC.LOCAL_FILE_SERVICE_KEY: (content_update, ) } self._write('content_updates', service_keys_to_content_updates) # HC.options['exclude_deleted_files'] = True result = self._read('md5_status', md5) self.assertEqual(result, ('deleted', None)) HC.options['exclude_deleted_files'] = False result = self._read('md5_status', md5) self.assertEqual(result, ('new', None))
def THREADDoFileQuery(self, query_key, search_context): try: query_hash_ids = self.Read('file_query_ids', search_context) query_hash_ids = list(query_hash_ids) random.shuffle(query_hash_ids) limit = search_context.GetSystemPredicates().GetLimit() if limit is not None: query_hash_ids = query_hash_ids[:limit] service_key = search_context.GetFileServiceKey() include_current_tags = search_context.IncludeCurrentTags() media_results = [] include_pending_tags = search_context.IncludePendingTags() i = 0 base = 256 while i < len(query_hash_ids): if query_key.IsCancelled(): return if i == 0: (last_i, i) = (0, base) else: (last_i, i) = (i, i + base) sub_query_hash_ids = query_hash_ids[last_i:i] more_media_results = self.Read('media_results_from_ids', service_key, sub_query_hash_ids) media_results.extend(more_media_results) HC.pubsub.pub('set_num_query_results', len(media_results), len(query_hash_ids)) self.WaitUntilGoodTimeToUseGUIThread() HC.pubsub.pub('file_query_done', query_key, media_results) except Exception as e: HC.ShowException(e)
def _RefreshConnection(self): if self._scheme == 'http': self._connection = httplib.HTTPConnection(self._host, self._port, timeout=self._timeout) elif self._scheme == 'https': self._connection = httplib.HTTPSConnection(self._host, self._port, timeout=self._timeout) try: self._connection.connect() except: raise Exception('Could not connect to ' + HC.u(self._host) + '!')
def BuildSimpleChildrenToParents(pairs): simple_children_to_parents = HC.default_dict_set() for (child, parent) in pairs: if child == parent: continue if LoopInSimpleChildrenToParents(simple_children_to_parents, child, parent): continue simple_children_to_parents[child].add(parent) return simple_children_to_parents
def test_4chan_pass(self): result = self._read('4chan_pass') self.assertTrue(result, ('', '', 0)) token = 'token' pin = 'pin' timeout = HC.GetNow() + 100000 self._write('4chan_pass', (token, pin, timeout)) result = self._read('4chan_pass') self.assertTrue(result, (token, pin, timeout))
def _threadDoPOSTJob(self, request): account = request.hydrus_account account_key = account.GetAccountKey() file_dict = request.hydrus_args file_dict['ip'] = request.getClientIP() HC.app.Write('file', self._service_key, account_key, file_dict) response_context = HC.ResponseContext(200) return response_context
def StartServer(*args, **kwargs): try: connection = httplib.HTTPConnection('127.0.0.1', port, timeout=10) try: connection.connect() connection.close() text = 'The client\'s local server could not start because something was already bound to port ' + HC.u( port) + '.' text += os.linesep * 2 text += 'This usually means another hydrus client is already running and occupying that port. It could be a previous instantiation of this client that has yet to shut itself down.' text += os.linesep * 2 text += 'You can change the port this client tries to host its local server on in file->options.' wx.CallLater(1, HC.ShowText, text) except: self._local_service = reactor.listenTCP( port, HydrusServer.HydrusServiceLocal( HC.LOCAL_FILE_SERVICE_KEY, HC.LOCAL_FILE, 'This is the local file service.')) connection = httplib.HTTPConnection('127.0.0.1', port, timeout=10) try: connection.connect() connection.close() except: text = 'Tried to bind port ' + HC.u( port) + ' for the local server, but it failed.' wx.CallLater(1, HC.ShowText, text) except Exception as e: wx.CallAfter(HC.ShowException, e)
def _threadDoGETJob(self, request): subject_identifier = request.hydrus_args['subject_identifier'] subject_account_key = HC.app.Read('account_key_from_identifier', self._service_key, subject_identifier) account_info = HC.app.Read('account_info', self._service_key, subject_account_key) body = yaml.safe_dump({'account_info': account_info}) response_context = HC.ResponseContext(200, body=body) return response_context
def setUpClass(self): threading.Thread(target=reactor.run, kwargs={ 'installSignalHandlers': 0 }).start() services = [] self._local_service_identifier = HC.ServerServiceIdentifier( 'local file', HC.LOCAL_FILE) self._file_service_identifier = HC.ServerServiceIdentifier( 'file service', HC.FILE_REPOSITORY) self._tag_service_identifier = HC.ServerServiceIdentifier( 'tag service', HC.TAG_REPOSITORY) permissions = [ HC.GET_DATA, HC.POST_DATA, HC.POST_PETITIONS, HC.RESOLVE_PETITIONS, HC.MANAGE_USERS, HC.GENERAL_ADMIN, HC.EDIT_SERVICES ] account_id = 1 account_type = HC.AccountType('account', permissions, (None, None)) created = HC.GetNow() - 100000 expires = None used_data = (0, 0) self._account = HC.Account(account_id, account_type, created, expires, used_data) self._access_key = os.urandom(32) self._file_hash = os.urandom(32) def TWISTEDSetup(): reactor.listenTCP( HC.DEFAULT_SERVER_ADMIN_PORT, HydrusServer.HydrusServiceAdmin(HC.SERVER_ADMIN_IDENTIFIER, 'hello')) reactor.listenTCP( HC.DEFAULT_LOCAL_FILE_PORT, HydrusServer.HydrusServiceLocal(self._local_service_identifier, 'hello')) reactor.listenTCP( HC.DEFAULT_SERVICE_PORT, HydrusServer.HydrusServiceRepositoryFile( self._file_service_identifier, 'hello')) reactor.listenTCP( HC.DEFAULT_SERVICE_PORT + 1, HydrusServer.HydrusServiceRepositoryTag( self._tag_service_identifier, 'hello')) reactor.callFromThread(TWISTEDSetup) time.sleep(1)
def GetAccount(self, service_key, session_key): with self._lock: service_sessions = self._service_keys_to_sessions[service_key] if session_key in service_sessions: (account, expires) = service_sessions[session_key] now = HC.GetNow() if now > expires: del service_sessions[session_key] else: return account raise HydrusExceptions.SessionException()
def _recordDataUsage(self, request): p1 = request.method == 'GET' and self.RECORD_GET_DATA_USAGE p2 = request.method == 'POST' and self.RECORD_POST_DATA_USAGE if p1 or p2: num_bytes = request.hydrus_request_data_usage HC.pubsub.pub( 'service_updates_delayed', { HC.LOCAL_BOORU_SERVICE_KEY: [ HC.ServiceUpdate(HC.SERVICE_UPDATE_REQUEST_MADE, num_bytes) ] })