def test_news(self):
result = self._read('news', HC.LOCAL_TAG_SERVICE_KEY)
self.assertEqual(result, [])
#
news = []
news.append(('hello', HC.GetNow() - 30000))
news.append(('hello again', HC.GetNow() - 20000))
service_updates = dict()
service_updates[HC.LOCAL_TAG_SERVICE_KEY] = [
HC.ServiceUpdate(HC.SERVICE_UPDATE_NEWS, news)
]
self._write('service_updates', service_updates)
#
result = self._read('news', HC.LOCAL_TAG_SERVICE_KEY)
self.assertItemsEqual(result, news)
def test_sessions(self):
result = self._read('hydrus_sessions')
self.assertEqual(result, [])
session = (HC.LOCAL_FILE_SERVICE_KEY, os.urandom(32),
HC.GetNow() + 100000)
self._write('hydrus_session', *session)
result = self._read('hydrus_sessions')
self.assertEqual(result, [session])
#
result = self._read('web_sessions')
self.assertEqual(result, [])
session = ('website name', ['cookie 1',
'cookie 2'], HC.GetNow() + 100000)
self._write('web_session', *session)
result = self._read('web_sessions')
self.assertEqual(result, [session])
def DAEMONMaintainConnections(self):
while True:
if HC.shutdown: break
last_checked = 0
if HC.GetNow() - last_checked > 30:
with self._lock:
connections_copy = dict(self._connections)
for (location, connection) in connections_copy.items():
with connection.lock:
if connection.IsStale():
del self._connections[location]
last_checked = HC.GetNow()
time.sleep(1)
def test_media_results(self):
self._clear_db()
hash = '\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
md5 = 'fdadb2cae78f2dfeb629449cd005f2a2'.decode('hex')
path = HC.STATIC_DIR + os.path.sep + 'hydrus.png'
self._write('import_file', path)
#
(media_result, ) = self._read('media_results',
HC.LOCAL_FILE_SERVICE_KEY, (hash, ))
(mr_hash, mr_inbox, mr_size, mr_mime, mr_timestamp, mr_width,
mr_height, mr_duration, mr_num_frames, mr_num_words, mr_tags_manager,
mr_locations_manager, mr_local_ratings,
mr_remote_ratings) = media_result.ToTuple()
now = HC.GetNow()
self.assertEqual(mr_hash, hash)
self.assertEqual(mr_inbox, True)
self.assertEqual(mr_size, 5270)
self.assertEqual(mr_mime, HC.IMAGE_PNG)
self.assertEqual(mr_hash, hash)
self.assertLessEqual(now - 10, mr_timestamp)
self.assertLessEqual(mr_timestamp, now + 10)
self.assertEqual(mr_width, 200)
self.assertEqual(mr_height, 200)
self.assertEqual(mr_duration, None)
self.assertEqual(mr_num_frames, None)
self.assertEqual(mr_num_words, None)
(media_result, ) = self._read('media_results_from_ids',
HC.LOCAL_FILE_SERVICE_KEY, (1, ))
(mr_hash, mr_inbox, mr_size, mr_mime, mr_timestamp, mr_width,
mr_height, mr_duration, mr_num_frames, mr_num_words, mr_tags_manager,
mr_locations_manager, mr_local_ratings,
mr_remote_ratings) = media_result.ToTuple()
now = HC.GetNow()
self.assertEqual(mr_hash, hash)
self.assertEqual(mr_inbox, True)
self.assertEqual(mr_size, 5270)
self.assertEqual(mr_mime, HC.IMAGE_PNG)
self.assertEqual(mr_hash, hash)
self.assertLessEqual(now - 10, mr_timestamp)
self.assertLessEqual(mr_timestamp, now + 10)
self.assertEqual(mr_width, 200)
self.assertEqual(mr_height, 200)
self.assertEqual(mr_duration, None)
self.assertEqual(mr_num_frames, None)
self.assertEqual(mr_num_words, None)
def AddSession(self, service_key, access_key):
with self._lock:
account_key = HC.app.Read('account_key_from_access_key',
service_key, access_key)
if account_key not in self._account_keys_to_accounts:
account = HC.app.Read('account', account_key)
self._account_keys_to_accounts[account_key] = account
account = self._account_keys_to_accounts[account_key]
session_key = os.urandom(32)
self._account_keys_to_session_keys[account_key].add(session_key)
now = HC.GetNow()
expires = now + HYDRUS_SESSION_LIFETIME
HC.app.Write('session', session_key, service_key, account_key,
expires)
self._service_keys_to_sessions[service_key][session_key] = (
account, expires)
return (session_key, expires)
def setUpClass(self):
threading.Thread(target=reactor.run,
kwargs={
'installSignalHandlers': 0
}).start()
services = []
self._local_service_identifier = HC.ServerServiceIdentifier(
'local file', HC.LOCAL_FILE)
self._file_service_identifier = HC.ServerServiceIdentifier(
'file service', HC.FILE_REPOSITORY)
self._tag_service_identifier = HC.ServerServiceIdentifier(
'tag service', HC.TAG_REPOSITORY)
permissions = [
HC.GET_DATA, HC.POST_DATA, HC.POST_PETITIONS, HC.RESOLVE_PETITIONS,
HC.MANAGE_USERS, HC.GENERAL_ADMIN, HC.EDIT_SERVICES
]
account_id = 1
account_type = HC.AccountType('account', permissions, (None, None))
created = HC.GetNow() - 100000
expires = None
used_data = (0, 0)
self._account = HC.Account(account_id, account_type, created, expires,
used_data)
self._access_key = os.urandom(32)
self._file_hash = os.urandom(32)
def TWISTEDSetup():
reactor.listenTCP(
HC.DEFAULT_SERVER_ADMIN_PORT,
HydrusServer.HydrusServiceAdmin(HC.SERVER_ADMIN_IDENTIFIER,
'hello'))
reactor.listenTCP(
HC.DEFAULT_LOCAL_FILE_PORT,
HydrusServer.HydrusServiceLocal(self._local_service_identifier,
'hello'))
reactor.listenTCP(
HC.DEFAULT_SERVICE_PORT,
HydrusServer.HydrusServiceRepositoryFile(
self._file_service_identifier, 'hello'))
reactor.listenTCP(
HC.DEFAULT_SERVICE_PORT + 1,
HydrusServer.HydrusServiceRepositoryTag(
self._tag_service_identifier, 'hello'))
reactor.callFromThread(TWISTEDSetup)
time.sleep(1)
def TIMEREventMaintenance(self, event):
sys.stdout.flush()
sys.stderr.flush()
self._CheckIfJustWokeFromSleep()
self._timestamps['last_maintenance_time'] = HC.GetNow()
if not self._just_woke_from_sleep and self.CurrentlyIdle():
self.MaintainDB()
def _CheckIfJustWokeFromSleep(self):
last_maintenance_time = self._timestamps['last_maintenance_time']
if last_maintenance_time == 0: return False
# this tests if we probably just woke up from a sleep
if HC.GetNow() - last_maintenance_time > MAINTENANCE_PERIOD + (5 * 60):
self._just_woke_from_sleep = True
else:
self._just_woke_from_sleep = False
def __init__(self, location, long_timeout=False):
(self._scheme, self._host, self._port) = location
if long_timeout: self._timeout = 600
else: self._timeout = 30
self.lock = threading.Lock()
self._last_request_time = HC.GetNow()
self._RefreshConnection()
def MaintainDB(self):
gc.collect()
now = HC.GetNow()
shutdown_timestamps = self.Read('shutdown_timestamps')
if HC.options['maintenance_vacuum_period'] != 0:
if now - shutdown_timestamps[
CC.SHUTDOWN_TIMESTAMP_VACUUM] > HC.options[
'maintenance_vacuum_period']:
self.Write('vacuum')
if HC.options['maintenance_delete_orphans_period'] != 0:
if now - shutdown_timestamps[
CC.SHUTDOWN_TIMESTAMP_DELETE_ORPHANS] > HC.options[
'maintenance_delete_orphans_period']:
self.Write('delete_orphans')
if now - self._timestamps['last_service_info_cache_fatten'] > 60 * 20:
HC.pubsub.pub('set_splash_text', 'fattening service info')
services = self.GetManager('services').GetServices()
for service in services:
try:
self.Read('service_info', service.GetServiceKey())
except:
pass # sometimes this breaks when a service has just been removed and the client is closing, so ignore the error
self._timestamps['last_service_info_cache_fatten'] = HC.GetNow()
HC.pubsub.pub('clear_closed_pages')
def test_4chan_pass(self):
result = self._read('4chan_pass')
self.assertTrue(result, ('', '', 0))
token = 'token'
pin = 'pin'
timeout = HC.GetNow() + 100000
self._write('4chan_pass', (token, pin, timeout))
result = self._read('4chan_pass')
self.assertTrue(result, (token, pin, timeout))
def GetAccount(self, service_key, session_key):
with self._lock:
service_sessions = self._service_keys_to_sessions[service_key]
if session_key in service_sessions:
(account, expires) = service_sessions[session_key]
now = HC.GetNow()
if now > expires: del service_sessions[session_key]
else: return account
raise HydrusExceptions.SessionException()
def test_import(self):
self._clear_db()
hash = '\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
path = HC.STATIC_DIR + os.path.sep + 'hydrus.png'
generate_media_result = True
(written_result, written_hash) = self._write('import_file', path)
self.assertEqual(written_result, 'successful')
self.assertEqual(written_hash, hash)
(written_result,
written_media_result) = self._write('import_file',
path,
generate_media_result=True)
self.assertEqual(written_result, 'redundant')
(mr_hash, mr_inbox, mr_size, mr_mime, mr_timestamp, mr_width,
mr_height, mr_duration, mr_num_frames, mr_num_words, mr_tags_manager,
mr_locations_manager, mr_local_ratings,
mr_remote_ratings) = written_media_result.ToTuple()
now = HC.GetNow()
self.assertEqual(mr_hash, hash)
self.assertEqual(mr_inbox, True)
self.assertEqual(mr_size, 5270)
self.assertEqual(mr_mime, HC.IMAGE_PNG)
self.assertEqual(mr_hash, hash)
self.assertLessEqual(now - 10, mr_timestamp)
self.assertLessEqual(mr_timestamp, now + 10)
self.assertEqual(mr_width, 200)
self.assertEqual(mr_height, 200)
self.assertEqual(mr_duration, None)
self.assertEqual(mr_num_frames, None)
self.assertEqual(mr_num_words, None)
def AddSession(self, service_key, access_key, name):
session_key = os.urandom(32)
account_key = HC.app.Read('account_key_from_access_key', service_key,
access_key)
account = HC.app.Read('account', service_key, account_key)
now = HC.GetNow()
expires = now + HYDRUS_SESSION_LIFETIME
with self._lock:
self._service_keys_to_sessions[service_key][session_key] = (
account, name, expires)
HC.app.Write('messaging_session', service_key, session_key,
account_key, name, expires)
return session_key
def _threadDoGETJob(self, request):
access_key = self._parseAccessKey(request)
session_manager = HC.app.GetManager('restricted_services_sessions')
(session_key,
expires) = session_manager.AddSession(self._service_key, access_key)
now = HC.GetNow()
max_age = now - expires
cookies = [('session_key', session_key.encode('hex'), {
'max_age': max_age,
'path': '/'
})]
response_context = HC.ResponseContext(200, cookies=cookies)
return response_context
def GetIdentityAndName(self, service_key, session_key):
with self._lock:
if session_key not in self._service_keys_to_sessions[service_key]:
raise HydrusExceptions.SessionException(
'Did not find that session!')
else:
(account, name, expires
) = self._service_keys_to_sessions[service_key][session_key]
now = HC.GetNow()
if now > expires:
del self._service_keys_to_sessions[service_key][
session_key]
raise HydrusExceptions.SessionException('Session expired!')
return (account.GetAccountKey(), name)
def GetSessionKey(self, service_key):
now = HC.GetNow()
with self._lock:
if service_key in self._service_keys_to_sessions:
(session_key,
expires) = self._service_keys_to_sessions[service_key]
if now + 600 > expires:
del self._service_keys_to_sessions[service_key]
else:
return session_key
# session key expired or not found
service = HC.app.GetManager('services').GetService(service_key)
(response, cookies) = service.Request(HC.GET,
'session_key',
return_cookies=True)
try:
session_key = cookies['session_key'].decode('hex')
except:
raise Exception('Service did not return a session key!')
expires = now + HYDRUS_SESSION_LIFETIME
self._service_keys_to_sessions[service_key] = (session_key,
expires)
HC.app.Write('hydrus_session', service_key, session_key, expires)
return session_key
def OnInit(self):
self.SetAssertMode(wx.PYAPP_ASSERT_SUPPRESS)
HC.app = self
HC.http = HydrusNetworking.HTTPConnectionManager()
self._timestamps = collections.defaultdict(lambda: 0)
self._timestamps['boot'] = HC.GetNow()
self._just_woke_from_sleep = False
self._local_service = None
self._booru_service = None
self.Bind(HC.EVT_PUBSUB, self.EventPubSub)
try:
splash = ClientGUI.FrameSplash('boot')
return True
except:
print('There was an error trying to start the splash screen!')
print(traceback.format_exc())
try:
wx.CallAfter(splash.Destroy)
except:
pass
return False
def _test_file_repo(self, service, host, port):
info = service.GetInfo()
info['access_key'] = self._access_key
# file
path = SC.GetExpectedPath('file', self._file_hash)
with open(path, 'wb') as f:
f.write('file')
response = service.Request(HC.GET, 'file',
{'hash': self._file_hash.encode('hex')})
self.assertEqual(response, 'file')
try:
os.remove(path)
except:
pass
path = HC.STATIC_DIR + os.path.sep + 'hydrus.png'
with open(path, 'rb') as f:
file = f.read()
service.Request(HC.POST, 'file', {'file': file})
written = HC.app.GetWrite('file')
[(args, kwargs)] = written
(written_service_key, written_account, written_file_dict) = args
self.assertEqual(
written_file_dict['hash'],
'\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
)
self.assertEqual(written_file_dict['ip'], '127.0.0.1')
self.assertEqual(written_file_dict['height'], 200)
self.assertEqual(written_file_dict['width'], 200)
self.assertEqual(written_file_dict['mime'], 2)
self.assertEqual(written_file_dict['size'], 5270)
# ip
(ip, timestamp) = ('94.45.87.123', HC.GetNow() - 100000)
HC.app.SetRead('ip', (ip, timestamp))
response = service.Request(HC.GET, 'ip',
{'hash': self._file_hash.encode('hex')})
self.assertEqual(response['ip'], ip)
self.assertEqual(response['timestamp'], timestamp)
# thumbnail
path = SC.GetExpectedPath('thumbnail', self._file_hash)
with open(path, 'wb') as f:
f.write('thumb')
response = service.Request(HC.GET, 'thumbnail',
{'hash': self._file_hash.encode('hex')})
self.assertEqual(response, 'thumb')
try:
os.remove(path)
except:
pass
def GetCookies(self, name):
now = HC.GetNow()
with self._lock:
if name in self._names_to_sessions:
(cookies, expires) = self._names_to_sessions[name]
if now + 300 > expires: del self._names_to_sessions[name]
else: return cookies
# name not found, or expired
if name == 'hentai foundry':
(response_gumpf, cookies) = HC.http.Request(
HC.GET,
'http://www.hentai-foundry.com/?enterAgree=1',
return_cookies=True)
expires = now + 60 * 60
elif name == 'pixiv':
(id, password) = HC.app.Read('pixiv_account')
if id == '' and password == '':
raise Exception(
'You need to set up your pixiv credentials in services->manage pixiv account.'
)
form_fields = {}
form_fields['mode'] = 'login'
form_fields['pixiv_id'] = id
form_fields['pass'] = password
form_fields['skip'] = '1'
body = urllib.urlencode(form_fields)
headers = {}
headers['Content-Type'] = 'application/x-www-form-urlencoded'
(response_gumpf,
cookies) = HC.http.Request(HC.POST,
'http://www.pixiv.net/login.php',
request_headers=headers,
body=body,
return_cookies=True)
# _ only given to logged in php sessions
if 'PHPSESSID' not in cookies or '_' not in cookies[
'PHPSESSID']:
raise Exception('Pixiv login credentials not accepted!')
expires = now + 30 * 86400
self._names_to_sessions[name] = (cookies, expires)
HC.app.Write('web_session', name, cookies, expires)
return cookies
def CurrentlyIdle(self):
if HC.options['idle_period'] == 0: return False
return HC.GetNow(
) - self._timestamps['last_user_action'] > HC.options['idle_period']
def ResetIdleTimer(self):
self._timestamps['last_user_action'] = HC.GetNow()
def IsStale(self):
time_since_last_request = HC.GetNow() - self._last_request_time
return time_since_last_request > self._timeout
def Request(self,
method,
path_and_query,
request_headers,
body,
report_hooks=[],
response_to_path=False):
if method == HC.GET: method_string = 'GET'
elif method == HC.POST: method_string = 'POST'
if 'User-Agent' not in request_headers:
request_headers['User-Agent'] = 'hydrus/' + HC.u(
HC.NETWORK_VERSION)
# it is important to only send str, not unicode, to httplib
# it uses += to extend the message body, which propagates the unicode (and thus fails) when
# you try to push non-ascii bytes as the body (e.g. during a file upload!)
method_string = str(method_string)
path_and_query = str(path_and_query)
request_headers = {
str(k): str(v)
for (k, v) in request_headers.items()
}
try:
self._connection.request(method_string,
path_and_query,
headers=request_headers,
body=body)
response = self._connection.getresponse()
except (httplib.CannotSendRequest, httplib.BadStatusLine):
# for some reason, we can't send a request on the current connection, so let's make a new one and try again!
self._RefreshConnection()
self._connection.request(method_string,
path_and_query,
headers=request_headers,
body=body)
response = self._connection.getresponse()
if response.status == 200 and response_to_path:
(temp_path, size_of_response) = self._WriteResponseToPath(
response, report_hooks)
parsed_response = temp_path
else:
(parsed_response,
size_of_response) = self._ParseResponse(response, report_hooks)
response_headers = {
k: v
for (k, v) in response.getheaders() if k != 'set-cookie'
}
cookies = self._ParseCookies(response.getheader('set-cookie'))
self._last_request_time = HC.GetNow()
if response.status == 200:
return (parsed_response, None, size_of_response, response_headers,
cookies)
elif response.status in (301, 302, 303, 307):
location = response.getheader('Location')
if location is None: raise Exception(parsed_response)
else:
url = location
if ' ' in url:
# some booru is giving daft redirect responses
print(url)
url = urllib.quote(url, safe='/?=&')
print(url)
if not url.startswith(self._scheme):
# assume it is like 'index.php' or '/index.php', rather than 'http://blah.com/index.php'
if url.startswith('/'): slash_sep = ''
else: slash_sep = '/'
url = self._scheme + '://' + self._host + slash_sep + url
if response.status in (301, 307):
# 301: moved permanently, repeat request
# 307: moved temporarily, repeat request
redirect_info = (method, url)
elif response.status in (302, 303):
# 302: moved temporarily, repeat request (except everyone treats it like 303 for no good f*****g reason)
# 303: thanks, now go here with GET
redirect_info = (HC.GET, url)
return (parsed_response, redirect_info, size_of_response,
response_headers, cookies)
elif response.status == 304:
raise HydrusExceptions.NotModifiedException()
else:
if response.status == 401:
raise HydrusExceptions.PermissionException(parsed_response)
elif response.status == 403:
raise HydrusExceptions.ForbiddenException(parsed_response)
elif response.status == 404:
raise HydrusExceptions.NotFoundException(parsed_response)
elif response.status == 426:
raise HydrusExceptions.NetworkVersionException(parsed_response)
elif response.status in (500, 501, 502, 503):
raise Exception(parsed_response)
else:
raise Exception(parsed_response)
def __init__(self, system_predicates):
self._predicates = {}
self._predicates[self.NUM_ATTACHMENTS] = []
self._status = None
self._contact_from = None
self._contact_to = None
self._contact_started = None
self._min_timestamp = None
self._max_timestamp = None
self._inbox = 'system:inbox' in system_predicates
self._archive = 'system:archive' in system_predicates
self._draft = 'system:draft' in system_predicates
isin = lambda a, b: a in b
startswith = lambda a, b: a.startswith(b)
lessthan = lambda a, b: a < b
greaterthan = lambda a, b: a > b
equals = lambda a, b: a == b
about_equals = lambda a, b: a < b * 1.15 and a > b * 0.85
for predicate in system_predicates:
if predicate.startswith('system:status'):
try:
status = predicate[14:]
self._status = status
except:
raise Exception('I could not parse the status predicate.')
if predicate.startswith('system:started_by'):
try:
started_by = predicate[18:]
self._contact_started = started_by
except:
raise Exception(
'I could not parse the started by predicate.')
if predicate.startswith('system:from'):
try:
contact_from = predicate[12:]
self._contact_from = contact_from
except:
raise Exception(
'I could not parse the contact from predicate.')
if predicate.startswith('system:to'):
try:
contact_to = predicate[10:]
self._contact_to = contact_to
except:
raise Exception(
'I could not parse the contact to predicate.')
if predicate.startswith('system:age'):
try:
condition = predicate[10]
if condition not in ('<', '>', u'\u2248'):
raise Exception()
age = predicate[11:]
years = 0
months = 0
days = 0
if 'y' in age:
(years, age) = age.split('y')
years = int(years)
if 'm' in age:
(months, age) = age.split('m')
months = int(months)
if 'd' in age:
(days, age) = age.split('d')
days = int(days)
timestamp = HC.GetNow() - ((((
(years * 12) + months) * 30) + days) * 86400)
# this is backwards because we are talking about age, not timestamp
if condition == '<': self._max_timestamp = timestamp
elif condition == '>': self._min_timestamp = timestamp
elif condition == u'\u2248':
self._min_timestamp = int(timestamp * 0.85)
self._max_timestamp = int(timestamp * 1.15)
except:
raise Exception('I could not parse the age predicate.')
if predicate.startswith('system:numattachments'):
try:
condition = predicate[21]
if condition not in ('>', '<', '=', u'\u2248'):
raise Exception()
num_attachments = int(predicate[22:])
if num_attachments >= 0:
if condition == '<':
self._predicates[self.NUM_ATTACHMENTS].append(
(lessthan, num_attachments))
elif condition == '>':
self._predicates[self.NUM_ATTACHMENTS].append(
(greaterthan, num_attachments))
elif condition == '=':
self._predicates[self.NUM_ATTACHMENTS].append(
(equals, num_attachments))
elif condition == u'\u2248':
self._predicates[self.NUM_ATTACHMENTS].append(
(about_equals, num_attachments))
except:
raise Exception(
'I could not parse the num attachments predicate.')
def _test_file_repo(self, host, port):
# set up connection
service_identifier = HC.ClientServiceIdentifier(
os.urandom(32), HC.FILE_REPOSITORY, 'service')
credentials = CC.Credentials(host, port, self._access_key)
connection = CC.ConnectionToService(service_identifier, credentials)
file_connection = httplib.HTTPConnection(host, port, timeout=10)
# file
path = SC.GetExpectedPath('file', self._file_hash)
with open(path, 'wb') as f:
f.write('file')
response = connection.Get('file', hash=self._file_hash.encode('hex'))
self.assertEqual(response, 'file')
try:
os.remove(path)
except:
pass
path = HC.STATIC_DIR + os.path.sep + 'hydrus.png'
with open(path, 'rb') as f:
file = f.read()
connection.Post('file', file=file)
written = HC.app.GetWrite('file')
[(args, kwargs)] = written
(written_service_identifier, written_account, written_file_dict) = args
self.assertEqual(
written_file_dict['hash'],
'\xadm5\x99\xa6\xc4\x89\xa5u\xeb\x19\xc0&\xfa\xce\x97\xa9\xcdey\xe7G(\xb0\xce\x94\xa6\x01\xd22\xf3\xc3'
)
self.assertEqual(written_file_dict['ip'], '127.0.0.1')
self.assertEqual(written_file_dict['height'], 200)
self.assertEqual(written_file_dict['width'], 200)
self.assertEqual(written_file_dict['mime'], 2)
self.assertEqual(written_file_dict['size'], 5270)
# ip
(ip, timestamp) = ('94.45.87.123', HC.GetNow() - 100000)
HC.app.SetRead('ip', (ip, timestamp))
response = connection.Get('ip', hash=self._file_hash.encode('hex'))
self.assertEqual(response['ip'], ip)
self.assertEqual(response['timestamp'], timestamp)
# thumbnail
path = SC.GetExpectedPath('thumbnail', self._file_hash)
with open(path, 'wb') as f:
f.write('thumb')
response = connection.Get('thumbnail',
hash=self._file_hash.encode('hex'))
self.assertEqual(response, 'thumb')
try:
os.remove(path)
except:
pass
def test_import_folders_daemon( self ):
test_dir = HC.TEMP_DIR + os.path.sep + 'test'
if not os.path.exists( test_dir ): os.mkdir( test_dir )
with open( test_dir + os.path.sep + '1', 'wb' ) as f: f.write( TestConstants.tinest_gif )
with open( test_dir + os.path.sep + '2', 'wb' ) as f: f.write( TestConstants.tinest_gif )
with open( test_dir + os.path.sep + '3', 'wb' ) as f: f.write( TestConstants.tinest_gif )
with open( test_dir + os.path.sep + '4', 'wb' ) as f: f.write( 'blarg' ) # broken
with open( test_dir + os.path.sep + '5', 'wb' ) as f: f.write( TestConstants.tinest_gif ) # previously failed for whatever reason
#
path = test_dir
details = {}
details[ 'type' ] = HC.IMPORT_FOLDER_TYPE_SYNCHRONISE
details[ 'cached_imported_paths' ] = { test_dir + os.path.sep + '2' }
details[ 'failed_imported_paths' ] = { test_dir + os.path.sep + '5' }
details[ 'local_tag' ] = 'local tag'
details[ 'last_checked' ] = HC.GetNow() - 1500
details[ 'check_period' ] = 1000
old_details = dict( details )
HC.app.SetRead( 'import_folders', { path : details } )
ClientDB.DAEMONCheckImportFolders()
#(('C:\\code\\Hydrus\\temp\\7baa9a818a14b7a9cbefb04c16bdc45ac651eb7400c1996e66e2efeef9e3ee5d',), {'service_keys_to_tags': {HC.LOCAL_TAG_SERVICE_KEY: set(['local tag'])}})
#(('C:\\code\\Hydrus\\temp\\e0dbdcb1a13c0565ffb73f2f497528adbe1703ca1dfc69680202487187b9fcfa',), {'service_keys_to_tags': {HC.LOCAL_TAG_SERVICE_KEY: set(['local tag'])}})
#(('C:\\code\\Hydrus\\temp\\182c4eecf2a5b4dfc8b74813bcff5d967ed53d92a982d8ae18520e1504fa5902',), {'service_keys_to_tags': {HC.LOCAL_TAG_SERVICE_KEY: set(['local tag'])}})
import_file = HC.app.GetWrite( 'import_file' )
self.assertEqual( len( import_file ), 3 )
expected_tag_part = { 'service_keys_to_tags' : { HC.LOCAL_TAG_SERVICE_KEY : set( [ 'local tag' ] ) } }
( one, two, three ) = import_file
( temp_path, tag_part ) = one
self.assertEqual( tag_part, expected_tag_part )
( temp_path, tag_part ) = two
self.assertEqual( tag_part, expected_tag_part )
( temp_path, tag_part ) = three
self.assertEqual( tag_part, expected_tag_part )
# I need to expand tests here with the new file system
[ ( ( updated_path, updated_details ), kwargs ) ] = HC.app.GetWrite( 'import_folder' )
self.assertEqual( path, updated_path )
self.assertEqual( updated_details[ 'type' ], old_details[ 'type' ] )
self.assertEqual( updated_details[ 'cached_imported_paths' ], { test_dir + os.path.sep + '1', test_dir + os.path.sep + '2', test_dir + os.path.sep + '3' } )
self.assertEqual( updated_details[ 'failed_imported_paths' ], { test_dir + os.path.sep + '4', test_dir + os.path.sep + '5' } )
self.assertEqual( updated_details[ 'local_tag' ], old_details[ 'local_tag' ] )
self.assertGreater( updated_details[ 'last_checked' ], old_details[ 'last_checked' ] )
self.assertEqual( updated_details[ 'check_period' ], old_details[ 'check_period' ] )
#
path = test_dir
details = {}
details[ 'type' ] = HC.IMPORT_FOLDER_TYPE_DELETE
details[ 'cached_imported_paths' ] = set()
details[ 'failed_imported_paths' ] = { test_dir + os.path.sep + '5' }
details[ 'local_tag' ] = 'local tag'
details[ 'last_checked' ] = HC.GetNow() - 1500
details[ 'check_period' ] = 1000
old_details = dict( details )
HC.app.SetRead( 'import_folders', { path : details } )
ClientDB.DAEMONCheckImportFolders()
# improve these tests as above
# old entries
#(('GIF89a\x01\x00\x01\x00\x00\xff\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x00;',), {'service_keys_to_tags': {HC.LOCAL_TAG_SERVICE_KEY: set(['local tag'])}})
#(('GIF89a\x01\x00\x01\x00\x00\xff\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x00;',), {'service_keys_to_tags': {HC.LOCAL_TAG_SERVICE_KEY: set(['local tag'])}})
#(('GIF89a\x01\x00\x01\x00\x00\xff\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x00;',), {'service_keys_to_tags': {HC.LOCAL_TAG_SERVICE_KEY: set(['local tag'])}})
#(('blarg',), {'service_keys_to_tags': {HC.LOCAL_TAG_SERVICE_KEY: set(['local tag'])}})
import_file = HC.app.GetWrite( 'import_file' )
[ ( ( updated_path, updated_details ), kwargs ) ] = HC.app.GetWrite( 'import_folder' )
self.assertEqual( path, updated_path )
self.assertEqual( updated_details[ 'type' ], old_details[ 'type' ] )
self.assertEqual( updated_details[ 'cached_imported_paths' ], set() )
self.assertEqual( updated_details[ 'failed_imported_paths' ], { test_dir + os.path.sep + '4', test_dir + os.path.sep + '5' } )
self.assertEqual( updated_details[ 'local_tag' ], old_details[ 'local_tag' ] )
self.assertGreater( updated_details[ 'last_checked' ], old_details[ 'last_checked' ] )
self.assertEqual( updated_details[ 'check_period' ], old_details[ 'check_period' ] )
self.assertTrue( not os.path.exists( test_dir + os.path.sep + '1' ) )
self.assertTrue( not os.path.exists( test_dir + os.path.sep + '2' ) )
self.assertTrue( not os.path.exists( test_dir + os.path.sep + '3' ) )
self.assertTrue( os.path.exists( test_dir + os.path.sep + '4' ) )
self.assertTrue( os.path.exists( test_dir + os.path.sep + '5' ) )
os.remove( test_dir + os.path.sep + '4' )
os.remove( test_dir + os.path.sep + '5' )
os.rmdir( test_dir )
def setUpClass(self):
services = []
self._file_service = CC.Service(os.urandom(32), HC.FILE_REPOSITORY,
'file repo', {})
self._tag_service = CC.Service(os.urandom(32), HC.TAG_REPOSITORY,
'tag repo', {})
self._admin_service = CC.Service(os.urandom(32), HC.SERVER_ADMIN,
'server admin', {})
services_manager = HC.app.GetManager('services')
services_manager._keys_to_services[
self._file_service.GetServiceKey()] = self._file_service
services_manager._keys_to_services[
self._tag_service.GetServiceKey()] = self._tag_service
services_manager._keys_to_services[
self._admin_service.GetServiceKey()] = self._admin_service
permissions = [
HC.GET_DATA, HC.POST_DATA, HC.POST_PETITIONS, HC.RESOLVE_PETITIONS,
HC.MANAGE_USERS, HC.GENERAL_ADMIN, HC.EDIT_SERVICES
]
account_key = os.urandom(32)
account_type = HC.AccountType('account', permissions, (None, None))
created = HC.GetNow() - 100000
expires = None
used_bytes = 0
used_requests = 0
self._account = HC.Account(account_key, account_type, created, expires,
used_bytes, used_requests)
self._access_key = os.urandom(32)
self._file_hash = os.urandom(32)
def TWISTEDSetup():
reactor.listenTCP(
HC.DEFAULT_SERVER_ADMIN_PORT,
HydrusServer.HydrusServiceAdmin(
self._admin_service.GetServiceKey(), HC.SERVER_ADMIN,
'hello'))
reactor.listenTCP(
HC.DEFAULT_LOCAL_FILE_PORT,
HydrusServer.HydrusServiceLocal(HC.LOCAL_FILE_SERVICE_KEY,
HC.LOCAL_FILE, 'hello'))
reactor.listenTCP(
HC.DEFAULT_LOCAL_BOORU_PORT,
HydrusServer.HydrusServiceBooru(HC.LOCAL_BOORU_SERVICE_KEY,
HC.LOCAL_BOORU, 'hello'))
reactor.listenTCP(
HC.DEFAULT_SERVICE_PORT,
HydrusServer.HydrusServiceRepositoryFile(
self._file_service.GetServiceKey(), HC.FILE_REPOSITORY,
'hello'))
reactor.listenTCP(
HC.DEFAULT_SERVICE_PORT + 1,
HydrusServer.HydrusServiceRepositoryTag(
self._tag_service.GetServiceKey(), HC.TAG_REPOSITORY,
'hello'))
reactor.callFromThread(TWISTEDSetup)
time.sleep(1)
def test_server(self):
discard = HC.app.GetWrite(
'session') # just to discard gumph from testserver
session_key_1 = os.urandom(32)
service_key = os.urandom(32)
permissions = [
HC.GET_DATA, HC.POST_DATA, HC.POST_PETITIONS, HC.RESOLVE_PETITIONS,
HC.MANAGE_USERS, HC.GENERAL_ADMIN, HC.EDIT_SERVICES
]
access_key = os.urandom(32)
account_key = os.urandom(32)
account_type = HC.AccountType('account', permissions, (None, None))
created = HC.GetNow() - 100000
expires = HC.GetNow() + 300
used_bytes = 0
used_requests = 0
account = HC.Account(account_key, account_type, created, expires,
used_bytes, used_requests)
expires = HC.GetNow() - 10
HC.app.SetRead('sessions',
[(session_key_1, service_key, account, expires)])
session_manager = HydrusSessions.HydrusSessionManagerServer()
with self.assertRaises(HydrusExceptions.SessionException):
session_manager.GetAccount(service_key, session_key_1)
# test fetching a session already in db, after bootup
expires = HC.GetNow() + 300
HC.app.SetRead('sessions',
[(session_key_1, service_key, account, expires)])
session_manager = HydrusSessions.HydrusSessionManagerServer()
read_account = session_manager.GetAccount(service_key, session_key_1)
self.assertIs(read_account, account)
# test adding a session
expires = HC.GetNow() + 300
account_key_2 = os.urandom(32)
account_2 = HC.Account(account_key_2, account_type, created, expires,
used_bytes, used_requests)
HC.app.SetRead('account_key_from_access_key', account_key_2)
HC.app.SetRead('account', account_2)
(session_key_2,
expires_2) = session_manager.AddSession(service_key, access_key)
[(args, kwargs)] = HC.app.GetWrite('session')
(written_session_key, written_service_key, written_account_key,
written_expires) = args
self.assertEqual(
(session_key_2, service_key, account_key_2, expires_2),
(written_session_key, written_service_key, written_account_key,
written_expires))
read_account = session_manager.GetAccount(service_key, session_key_2)
self.assertIs(read_account, account_2)
# test adding a new session for an account already in the manager
HC.app.SetRead('account_key_from_access_key', account_key)
HC.app.SetRead('account', account)
(session_key_3,
expires_3) = session_manager.AddSession(service_key, access_key)
[(args, kwargs)] = HC.app.GetWrite('session')
(written_session_key, written_service_key, written_account_key,
written_expires) = args
self.assertEqual((session_key_3, service_key, account_key, expires_3),
(written_session_key, written_service_key,
written_account_key, written_expires))
read_account = session_manager.GetAccount(service_key, session_key_3)
self.assertIs(read_account, account)
read_account_original = session_manager.GetAccount(
service_key, session_key_1)
self.assertIs(read_account, read_account_original)
# test individual account refresh
expires = HC.GetNow() + 300
updated_account = HC.Account(account_key, account_type, created,
expires, 1, 1)
HC.app.SetRead('account', updated_account)
session_manager.RefreshAccounts(service_key, [account_key])
read_account = session_manager.GetAccount(service_key, session_key_1)
self.assertIs(read_account, updated_account)
read_account = session_manager.GetAccount(service_key, session_key_3)
self.assertIs(read_account, updated_account)
# test all account refresh
expires = HC.GetNow() + 300
updated_account_2 = HC.Account(account_key, account_type, created,
expires, 2, 2)
HC.app.SetRead(
'sessions',
[(session_key_1, service_key, updated_account_2, expires),
(session_key_2, service_key, account_2, expires),
(session_key_3, service_key, updated_account_2, expires)])
session_manager.RefreshAllAccounts()
read_account = session_manager.GetAccount(service_key, session_key_1)
self.assertIs(read_account, updated_account_2)
read_account = session_manager.GetAccount(service_key, session_key_2)
self.assertIs(read_account, account_2)
read_account = session_manager.GetAccount(service_key, session_key_3)
self.assertIs(read_account, updated_account_2)