def name_server_watch_poller_thread(self):
# every 10 minutes, retrieve and update the list of people watching me
while self.running:
pipe = self.node.retrieve(hash.hash_of('watch '+self.public_key_name),
settings.identity_redundancy)
watcher = {
'type' : 'watch',
'whom' : self.public_key_name
}
try:
while not pipe.finished():
for item in pipe.read_all():
if item[1] == watcher:
if item[0] not in self.watchers:
utility.start_thread(
name_server_add_watcher_thread(self,item[0]))
if not self.running:
return
# Ugh yuck bleah
yield 'sleep',2
except:
pass
pipe.stop()
yield 'sleep',10*60
def start(self):
self.last_update_time = time.time()
Downloader.start(self)
for i in range(download_threads):
utility.start_thread(circle_downloader_thread(self))
utility.start_thread(circle_downloader_refresh_sources_thread(self))
def download(self, app, dir=None,download_subdirs=0):
""" download a directory """
def download_thread(self, app, dirname, download_subdirs):
if not dirname:
dirname = app.daemon.config['download_dir']
dirname = os.path.join(dirname, self.item['path'][-1])
if not os.path.exists(dirname):
os.mkdir(dirname)
list = []
self.still_searching=1
sleep_time=0.1
while self.still_searching:
yield 'sleep',sleep_time
sleep_time=sleep_time*1.25
children = self.get_children()
for item in children:
if item not in list:
list.append(item)
if item.item['type']=='file':
app.file_server.download(item.item,item.sources,dirname,no_overload=1)
elif download_subdirs:
item.download(app,dirname,1)
self.update(app)
utility.start_thread(download_thread(self, app, dir, download_subdirs))
def start(self):
self.running = 1
self.node.add_handler('gossip list', self,
(types.IntType, types.IntType))
self.node.add_handler("gossip get",self)
if self.app.config.get('poll_gossip'):
utility.start_thread(gossip_poll_thread(self))
def send_message(self, dests, message, attachment=None, augmentation=None):
check.check_matches(dests, ['text'])
check.check_is_text(message)
if self.quiet:
self.show_status()
utility.start_thread(chat_tell_thread(
self,dests,message,attachment,augmentation))
def identify(self,key_name,address,on_complete):
def identify_thread(chat, key_name, address, on_complete):
try:
result_id, thread = chat.app.name_server.identify(key_name,address)
yield 'wait',thread
result = result_id[0]
except:
result = None
on_complete(result)
utility.start_thread(identify_thread(
self,key_name,address,on_complete))
def set_status(self, key,value):
self.lock.acquire()
try:
if not self.status.has_key(key) or \
self.status[key] != value:
self.status[key] = value
def id_change_thread(self, address, status):
try:
ticket, template, wait = self.node.call(
address,('identity status changed',self.public_key_name,status))
if wait: yield 'call',(self.node,ticket)
result = self.node.get_reply(ticket, template)
except error.Error:
pass
for item in self.watchers:
utility.start_thread(id_change_thread(self,item,self.status))
finally:
self.lock.release()
def start_watching(self, node):
if not self.running:
return
if not self.watching:
self.watcher = {
'type' : 'watch',
'whom' : self.name
}
name = self.name
watcher = self.watcher
self.watching = 1
node.publish(hash.hash_of('watch '+name), watcher, settings.identity_redundancy)
if not self.online:
self.watched = 0
utility.start_thread(acquaintance_watch_thread(self,node))
else:
self.watched = 1
def startup_thread(self, status_monitor=status_monitor):
list = self.acquaintances.values()
list.sort(lambda x,y: cmp(x.sort_value(),y.sort_value()))
for item in list:
item.start_watching(self.node)
#start watching tends to breed, try to make sure we don't get
#too many threads.
#yes, this is hacky
#print item.nickname, threading.activeCount()
#time.sleep(0.25)
while 1:
yield 'sleep',0.25
if threading.activeCount() < 40:
break
self.me.start_watching(self.node)
while not self.me.watched:
yield 'sleep',0.1
online = self.me.online
address = self.me.address
if online:
if status_monitor:
status_monitor(_('Shutting down your other peer.'))
while 1:
ticket,template,wait = self.node.call(address, ('identity abort',))
if wait: yield 'call',(self.node,ticket)
try:
dummy_result = self.node.get_reply(ticket, template)
except error.Error:
break
yield 'sleep',4
self.me.online = 1
self.me.address = self.node.address
self.me.connect_time = time.time()
# Task to retrieve existing watchers
# Task to poll existing watchers
utility.start_thread(name_server_watch_poller_thread(self))
# now refresh my own offline presence
pipe = self.node.retrieve(self.public_key_name_offline, settings.cache_redundancy)
list = [ ]
while not pipe.finished():
for item in pipe.read_all():
if type(item[1]) == types.DictType and \
item[1].get('type') == 'identity offline' and \
item[1].get('salt'):
list.append(item)
yield 'sleep',2
if not self.running:
return
pipe.stop()
#if len(list) != 4:
# print _("%d peers holding your offline presence.") % len(list)
for item in list:
address, value = item
key = hash.hash_of(safe_pickle.dumps(self.sign(value['salt'])))
ticket, template, wait = self.node.call(address, ('data cache remove',key))
if wait: yield 'call',(self.node,ticket)
try:
dummy_result = self.node.get_reply(ticket,template)
except error.Error:
pass
self.lock.acquire()
try:
package = {
'name' : self.info['name'],
'human-name' : self.info['human-name'],
'description': self.info['description'],
'timezone' : self.info['timezone'],
'key' : self.public_key,
'keywords' : self.info['keywords'],
}
finally:
self.lock.release()
package_dumped = safe_pickle.dumps(package)
signature = self.sign(package_dumped)
# now publish and cache offline identity
value = {
'type' : 'identity offline',
'package' : package_dumped,
'signature' : signature,
'salt' : utility.random_bytes(settings.name_bytes)
}
lock = hash.hash_of(hash.hash_of(safe_pickle.dumps(self.sign(value['salt']))))
publications = [ self.public_key_name_offline, self.service_name ]
for item in package['keywords']:
publications.append(hash.hash_of('identity-name '+item))
# thomasV
# redundancy 4: this is the meta-publish
result, publish_thread = self.app.cache.publish(publications,value,lock, 4)
yield 'wait',publish_thread
def chat_tell_thread(chat, dests, message, attachment=None, augmentation=None):
"""Messages are sent in parallel, one thread per recipient. The
information displayed in field ends with '...' until the last
message has been sent, then with '..' until all recipients
have answered, and finally with '.' The field is closed by
the last thread to finish """
check.check_matches(dests, ['text'])
check.check_is_text(message)
std_timestamp = host2standard_timestamp(time.time())
if augmentation and len(augmentation) != len(message):
sys.stderr.write("Augmentation %d bytes, should be %d bytes.\n"
% (len(augmentation), len(message)))
augmentation = None
message_id = chat.create_message_id()
try:
result = ['','','','','...\n']
dest_list = [ ]
dest_address = [ ]
checked_address = [ ]
dest_names = [ ]
received_names = [ ]
offline_list = [ ]
offline_names = [ ]
recipient_list = [ ]
channel_pipes = { }
chan_cnt=[0]
for dest in dests:
if dest[0]=='#':
channel_pipes[dest] = chat.channels.sub_list_pipe(dest)
recipient_list.append((None,None,dest))
continue
acq = chat.app.name_server.locate(dest)
acq.start_watching(chat.node)
while not acq.watched:
yield 'sleep',0.1
acq.lock.acquire()
online = acq.online
address = acq.address
name = acq.name
username = acq.info['name']
acq.lock.release()
if not online:
offline_list.append(acq)
offline_names.append(dest)
recipient_list.append((name,None,username))
else:
dest_list.append(acq)
dest_address.append(address)
dest_names.append(dest)
recipient_list.append((name,address,username))
my_details = (
chat.app.name_server.public_key_name,
chat.app.name_server.name
)
package = {
'from': my_details,
'to': recipient_list,
'text': message
}
if augmentation:
package['aug'] = augmentation
if attachment:
package['attach'] = attachment
package_pickle = safe_pickle.dumps(package)
signature = chat.app.name_server.sign(package_pickle)
message = ('chat message 2',package_pickle,signature)
def recipient_thread(chat,address,name,channel,received_names,dest_address,
checked_address,chan_cnt,message_id,result, message=message):
succ = 1
try:
ticket, template, wait = chat.node.call(address,message)
if wait: yield ('call',(chat.node,ticket))
ret_value = chat.node.get_reply(ticket,template)
#Type checking
if type(ret_value) != types.TupleType:
succ = 0
if not channel:
result[2] = result[2] + name + _(' sent bad reply.\n')
chat.update_message_status(message_id, result[0]+result[4]+result[2])
except error.Error,err:
succ = 0
if not channel:
result[2] = result[2] + name + _(' could not be contacted: ')+err.message+'\n'
chat.update_message_status(message_id, result[0]+result[4]+result[2])
chat.app.name_server.bad_address(address)
if succ:
if channel:
chan_cnt[0] += 1
else:
received_names.append(name)
if ret_value[0]:
pass
#if ret_value[1] != '':
# result[2] = result[2] + name + ' ' + ret_value[1] + '\n'
else:
if ret_value[1] == '':
result[2] = result[2] + name + _(' is quiet.\n')
else:
result[2] = result[2] + name + _(' is quiet: ') + ret_value[1] + '\n'
if chan_cnt[0] == 0:
result[0] = _('Message received by ') + utility.english_list(received_names)
elif chan_cnt[0] == 1:
result[0] = _('Message received by ')\
+utility.english_list(received_names+[ _('1 person')])+result[1]
else:
result[0] = _('Message received by ')\
+utility.english_list(received_names+[ _('%d people') % chan_cnt[0] ])+result[1]
chat.update_message_status(message_id,result[0]+result[4]+result[2])
checked_address.append(address)
checked_address.sort()
if result[4]=='..\n' and checked_address == dest_address:
if chan_cnt[0]==0 and received_names == []:
result[0] = _('Nobody received your message')
result[4] = '.\n'
chat.update_message_status(message_id, result[0]+result[4]+result[2])
for i in range(len(dest_list)):
utility.start_thread(recipient_thread(chat,dest_address[i],dest_names[i],0,\
received_names,dest_address,checked_address,chan_cnt,message_id,result))
if channel_pipes:
if channel_pipes.__len__()>1:
result[1] = (_(' on channels %s') % utility.english_list(channel_pipes.keys()))
else:
result[1] = (_(' on channel %s') % channel_pipes.keys()[0])
if channel_pipes:
for chan_name in channel_pipes.keys():
if not chat.channels.cache.has_key(chan_name):
chat.channels.cache[chan_name]=[]
else:
for address in chat.channels.cache[chan_name]:
if address in dest_address:
continue
dest_address.append(address)
utility.start_thread(
recipient_thread(chat,address,'',1,received_names,
dest_address,checked_address,chan_cnt,message_id,result))
#reset the cache:
chat.channels.cache[chan_name]=[]
while channel_pipes:
for chan_name,chan_pipe in channel_pipes.items():
if chan_pipe.finished():
chan_pipe.stop()
del channel_pipes[chan_name]
continue
for address in chan_pipe.read_all():
#update the cache
if address not in chat.channels.cache[chan_name]:
chat.channels.cache[chan_name].append(address)
if address in dest_address:
continue
dest_address.append(address)
utility.start_thread(recipient_thread(chat,address,'',1,received_names,
dest_address,checked_address,chan_cnt,message_id,result))
yield 'sleep',0.1
#we now have launched all the tasks
dest_address.sort()
result[4]='..\n'
if checked_address == dest_address:
if chan_cnt[0]==0 and received_names == []:
result[0] = _('Nobody received your message')
result[4] = '.\n'
chat.update_message_status(message_id, result[0]+result[4]+result[2])
recall_list = [ ]
if offline_list:
chat.update_message_status(message_id,'Caching message...\n',1)
for i in range(len(offline_list)):
key = utility.random_bytes(settings.name_bytes)
lock = hash.hash_of(key)
crypt = offline_list[i].encrypt((key, message, std_timestamp))
data = {
'type' : 'offline message',
'crypt': crypt
}
name = hash.hash_of('offline message '+offline_list[i].name)
# Loose proof of @R50: @I21, @E22.
recall_list.append((offline_list[i].nickname,name,data,key))
publish_result, subthread = chat.app.cache.publish([name],data,lock)
yield 'wait',subthread
if publish_result:
redundancy = publish_result[0]
else: redundancy = 0
if redundancy == settings.cache_redundancy:
str = offline_names[i] + _(' will get your message when next logged on.\n')
elif redundancy == 0:
str = _('Could not store message for ') + offline_names[i] + '!\n'
else:
str = (offline_names[i]
+ _(' will probably get your message when next logged on:\n' +\
' Message only stored with redundancy %d.\n') % redundancy)
result[3] = result[3] + str
chat.update_message_status(message_id,result[3]+'\n',1)
chat.recall_list = recall_list
def retrieve_cached_messages(self,on_complete):
utility.start_thread(retrieve_cached_messages_thread(self,on_complete))
def start(self):
self.cache = utility.get_config("hash_cache",{ })
utility.Task_manager.start(self)
self.node.add_handler('download chunk', self, ('name', 'integer', 'integer'), types.StringType)
self.node.add_handler('files available', self)
utility.start_thread(file_server_poll_thread(self))
def sync_gossip_thread(self):
def gossip_fetch_thread(self,acq):
self.fetch_threads+=1
if not self.running:
self.fetch_threads-=1
return
if acq.distance != None:
#acq.start_watching(self.node, 1)
acq.lock.acquire()
online = acq.online
address = acq.address
distance = acq.distance
acq.lock.release()
if distance == None or not online:
self.fetch_threads-=1
return
pos = 0
fetch_timeout = node.make_timeout(settings.gossip_fetch_time)
while not node.is_timed_out(fetch_timeout):
try:
ticket, template, wait = self.node.call(address,('gossip list',pos,pos+20))
if wait: yield 'call',(self.node,ticket)
result = self.node.get_reply(ticket,template)
if not check.matches(result,
[(types.IntType, 'any', 'any')]):
node.bad_peer(address,
_("Bad reply to 'gossip list': ") + `result`)
break
if len(result) == 0:
break
all_ok = 1
# effic: sort self.gossip, sort the returned results,
# to speed up searches for signatures. However, I
# haven't yet seen gossip_fetch_task come up in
# profiles.
for item in result:
already_there = 0
for wodge in self.gossip:
if wodge.signature == item[2]:
#if wodge.distance(self.app.name_server) > from_fixed(item[0])+distance:
# # TODO: What to do with unit_decay_time?
# wodge.initial_distance = from_fixed(item[0])+distance
# wodge.initial_time = time.time()
wodge.opinions[acq.name] = from_fixed(item[0]) - wodge.decay()
already_there = 1
break
if already_there:
continue
try:
ticket, template, wait = self.node.call(
address,('gossip get',item[2]))
if wait: yield 'call',(self.node,ticket)
string = self.node.get_reply(ticket, template)
wodgewodge = safe_pickle.loads(string)
except error.Error:
all_ok = 0
break
#TODO: Confirm signature of known people
wodge = Wodge({
'wodge': wodgewodge,
'string': string,
'signature': item[2],
'initial_time': time.time(),
'initial_distance': None,
#'initial_distance': from_fixed(item[0]) + distance
'unit_decay_time': item[1],
'opinions': { acq.name : from_fixed(item[0]) },
'collapsed': 0})
if not self.insert_wodge(wodge):
all_ok = 0
break
if not all_ok:
break
pos = pos + 18
except error.Error:
break
self.fetch_threads-=1
self.fetch_threads=0
for acq in self.app.name_server.acquaintances.values():
if acq.distance != None:
#acq.start_watching(self.node, 1)
#name server starts watching all
#want start watching to be gradual, see name server start()
while not acq.watched:
yield 'sleep',1
if not self.running:
return
if acq.online:
utility.start_thread(gossip_fetch_thread(self,acq))
while self.fetch_threads != 0:
yield 'sleep',0.1
requests = self.update_requests
self.update_requests = [ ]
for item in requests:
self.app.idle_add(item)
def handle(self, request, address,call_id):
check.check_matches(request, (types.StringType,))
check.check_is_af_inet_address(address) #=@R30
if request[0] == 'identity test':
check.check_matches(request[1:], ('name',))
return self.sign('identity test ' + request[1])
elif request[0] == 'identity query':
return self.get_info()
elif request[0] == 'identity watch':
self.lock.acquire()
try:
if address not in self.watchers:
self.watchers.append(address)
status = self.status
finally:
self.lock.release()
return status
elif request[0] == 'identity abort':
def abort_thread(self, address=address):
id_test_result = []
yield 'wait', identity_test_thread(address,self.public_key,self.node,id_test_result)
if not id_test_result[0]:
print "error testing identity"
return
self.aborted = 1
self.app.shutdown(_('Reconnecting from different machine'))
utility.start_thread(abort_thread(self))
return None
elif request[0] == 'identity connecting':
if not (len(request) >= 2):
return error.Error('Name_server.handle: identity connecting: expecting tuple of at least 2.')
self.acquire_lock('get acq')
try:
acq = self.acquaintances.get(request[1])
finally:
self.release_lock('get acq')
if acq is None:
return error.Error(_("I don't know you."))
else:
#if len(request) > 2:
# status = request[2]
#else:
# status = { }
def acq_change_thread(ns, acq, address, call_id, node):
check.check_is_af_inet_address(address) #=@R29
try:
result, subthread = try_address(acq,address,node)
yield 'wait',subthread
if not result[0]:
return
except error.Error:
return
#acq.lock.acquire()
#acq.status = status
#acq.lock.release()
ns.acquaintance_status_changed(acq, 'connect')
utility.start_thread(acq_change_thread(self, acq, address, call_id, self.node))
# Proof of @R29: @R30; deeply immutable by @E11.
return None
elif request[0] == 'identity status changed':
if not (len(request) >= 3):
return error.Error('Name_server.handle: identity status changed: expecting tuple of at least 3.')
self.lock.acquire()
try:
acq = self.acquaintances.get(request[1],None)
if acq:
acq.lock.acquire()
try:
acq.status = request[2]
finally:
acq.lock.release()
self.acquaintance_status_changed(acq, 'status changed')
finally:
self.lock.release()
elif request[0] == 'identity disconnecting':
if not (len(request) >= 2):
return error.Error('Name_server.handle: identity disconnecting: expecting tuple of at least 2.')
self.lock.acquire()
if address in self.watchers:
self.watchers.remove(address)
if self.acquaintances.has_key(request[1]):
acq = self.acquaintances[request[1]]
self.lock.release()
if acq.disconnection(address):
self.acquaintance_status_changed(acq, 'disconnect', message=request[2])
return None
else:
self.lock.release()
return error.Error(_("I don't know you."))
def request_update(self, callback):
self.update_requests.append(callback)
utility.start_thread(self.sync_gossip_thread())
def start(self, status_monitor=None):
utility.Task_manager.start(self)
acq_list = utility.get_checked_config('acquaintances', types.ListType, [ ])
for item in acq_list:
if not check.matches(item, Acquaintance.map_tmpl):
print _("Warning: corrupted acquaintances config file; ignoring item: "), item
continue
acq = Acquaintance(self, item, 1)
self.acquaintances[acq.name] = acq
self.nicknames[acq.nickname] = acq
acq.start()
def make_acquaintance_noaddr(self, info):
name = key_name(info['key'])
acq = Acquaintance(self, {'info': info,
'name': name,
'nickname': self.choose_nickname(info['name'])},0)
self.acquaintances[name] = acq
self.nicknames[acq.nickname] = acq
acq.start()
acq.start_watching(self.node)
self.acquaintance_status_changed(acq, "create")
return acq
self.me = make_acquaintance_noaddr(self,self.info)
# Other me may want to test identity
# May start chatting before test complete
self.node.add_handler('identity test', self, ('name',), crypto.pubkey.signature_tmpl)
self.node.add_handler('identity query', self, (), Acquaintance.info_template)
self.node.add_handler('identity watch', self, (), types.DictionaryType)
self.node.add_handler('identity connecting', self)
self.node.add_handler('identity status changed', self, ('any', Acquaintance.status_template))
self.node.add_handler('identity disconnecting', self,('string', 'opt-text'))
self.node.add_handler('identity abort', self)
self.node.publish(self.public_key_name,self.get_info_func, settings.identity_redundancy)
self.node.publish(self.public_key_name_offline,self.get_info_func, settings.identity_redundancy)
self.node.publish(self.service_name,self.get_info_func, settings.identity_redundancy)
for item in self.info['keywords']:
self.node.publish(hash.hash_of('identity-name '+item),
self.get_info_func, settings.identity_redundancy)
def startup_thread(self, status_monitor=status_monitor):
list = self.acquaintances.values()
list.sort(lambda x,y: cmp(x.sort_value(),y.sort_value()))
for item in list:
item.start_watching(self.node)
#start watching tends to breed, try to make sure we don't get
#too many threads.
#yes, this is hacky
#print item.nickname, threading.activeCount()
#time.sleep(0.25)
while 1:
yield 'sleep',0.25
if threading.activeCount() < 40:
break
self.me.start_watching(self.node)
while not self.me.watched:
yield 'sleep',0.1
online = self.me.online
address = self.me.address
if online:
if status_monitor:
status_monitor(_('Shutting down your other peer.'))
while 1:
ticket,template,wait = self.node.call(address, ('identity abort',))
if wait: yield 'call',(self.node,ticket)
try:
dummy_result = self.node.get_reply(ticket, template)
except error.Error:
break
yield 'sleep',4
self.me.online = 1
self.me.address = self.node.address
self.me.connect_time = time.time()
# Task to retrieve existing watchers
# Task to poll existing watchers
utility.start_thread(name_server_watch_poller_thread(self))
# now refresh my own offline presence
pipe = self.node.retrieve(self.public_key_name_offline, settings.cache_redundancy)
list = [ ]
while not pipe.finished():
for item in pipe.read_all():
if type(item[1]) == types.DictType and \
item[1].get('type') == 'identity offline' and \
item[1].get('salt'):
list.append(item)
yield 'sleep',2
if not self.running:
return
pipe.stop()
#if len(list) != 4:
# print _("%d peers holding your offline presence.") % len(list)
for item in list:
address, value = item
key = hash.hash_of(safe_pickle.dumps(self.sign(value['salt'])))
ticket, template, wait = self.node.call(address, ('data cache remove',key))
if wait: yield 'call',(self.node,ticket)
try:
dummy_result = self.node.get_reply(ticket,template)
except error.Error:
pass
self.lock.acquire()
try:
package = {
'name' : self.info['name'],
'human-name' : self.info['human-name'],
'description': self.info['description'],
'timezone' : self.info['timezone'],
'key' : self.public_key,
'keywords' : self.info['keywords'],
}
finally:
self.lock.release()
package_dumped = safe_pickle.dumps(package)
signature = self.sign(package_dumped)
# now publish and cache offline identity
value = {
'type' : 'identity offline',
'package' : package_dumped,
'signature' : signature,
'salt' : utility.random_bytes(settings.name_bytes)
}
lock = hash.hash_of(hash.hash_of(safe_pickle.dumps(self.sign(value['salt']))))
publications = [ self.public_key_name_offline, self.service_name ]
for item in package['keywords']:
publications.append(hash.hash_of('identity-name '+item))
# thomasV
# redundancy 4: this is the meta-publish
result, publish_thread = self.app.cache.publish(publications,value,lock, 4)
yield 'wait',publish_thread
utility.start_thread(startup_thread(self))
result = self.node.get_reply(ticket, template)
except error.Error,err:
#print "error on disconnecting",err,address
pass
except:
pass
self.disconnect_threads-=1
# self.watchers might change while we are disconnecting (especially due to 127.0.0.1)
# so we need to copy them before
if not self.aborted:
threads = []
for item in self.watchers:
threads.append(id_disconnect_thread(self,item))
for thread in threads:
utility.start_thread(thread)
self.watchers = [ ]
list = self.acquaintances.values()
self.change_activity('calling acq.stop_watching')
for item in list:
def stopper_task(self, item):
item.stop(self.node)
utility.Task(stopper_task, self, item).start()
#note that this might wait for all tasks: could be long
utility.Task_manager.stop(self)
self.change_activity('unpublishing self')
self.node.unpublish(self.get_info_func)
self.change_activity('removing handlers')
