def name_server_watch_poller_thread(self): # every 10 minutes, retrieve and update the list of people watching me while self.running: pipe = self.node.retrieve(hash.hash_of('watch '+self.public_key_name), settings.identity_redundancy) watcher = { 'type' : 'watch', 'whom' : self.public_key_name } try: while not pipe.finished(): for item in pipe.read_all(): if item[1] == watcher: if item[0] not in self.watchers: utility.start_thread( name_server_add_watcher_thread(self,item[0])) if not self.running: return # Ugh yuck bleah yield 'sleep',2 except: pass pipe.stop() yield 'sleep',10*60
def start(self): self.last_update_time = time.time() Downloader.start(self) for i in range(download_threads): utility.start_thread(circle_downloader_thread(self)) utility.start_thread(circle_downloader_refresh_sources_thread(self))
def download(self, app, dir=None,download_subdirs=0): """ download a directory """ def download_thread(self, app, dirname, download_subdirs): if not dirname: dirname = app.daemon.config['download_dir'] dirname = os.path.join(dirname, self.item['path'][-1]) if not os.path.exists(dirname): os.mkdir(dirname) list = [] self.still_searching=1 sleep_time=0.1 while self.still_searching: yield 'sleep',sleep_time sleep_time=sleep_time*1.25 children = self.get_children() for item in children: if item not in list: list.append(item) if item.item['type']=='file': app.file_server.download(item.item,item.sources,dirname,no_overload=1) elif download_subdirs: item.download(app,dirname,1) self.update(app) utility.start_thread(download_thread(self, app, dir, download_subdirs))
def start(self): self.running = 1 self.node.add_handler('gossip list', self, (types.IntType, types.IntType)) self.node.add_handler("gossip get",self) if self.app.config.get('poll_gossip'): utility.start_thread(gossip_poll_thread(self))
def send_message(self, dests, message, attachment=None, augmentation=None): check.check_matches(dests, ['text']) check.check_is_text(message) if self.quiet: self.show_status() utility.start_thread(chat_tell_thread( self,dests,message,attachment,augmentation))
def identify(self,key_name,address,on_complete): def identify_thread(chat, key_name, address, on_complete): try: result_id, thread = chat.app.name_server.identify(key_name,address) yield 'wait',thread result = result_id[0] except: result = None on_complete(result) utility.start_thread(identify_thread( self,key_name,address,on_complete))
def set_status(self, key,value): self.lock.acquire() try: if not self.status.has_key(key) or \ self.status[key] != value: self.status[key] = value def id_change_thread(self, address, status): try: ticket, template, wait = self.node.call( address,('identity status changed',self.public_key_name,status)) if wait: yield 'call',(self.node,ticket) result = self.node.get_reply(ticket, template) except error.Error: pass for item in self.watchers: utility.start_thread(id_change_thread(self,item,self.status)) finally: self.lock.release()
def start_watching(self, node): if not self.running: return if not self.watching: self.watcher = { 'type' : 'watch', 'whom' : self.name } name = self.name watcher = self.watcher self.watching = 1 node.publish(hash.hash_of('watch '+name), watcher, settings.identity_redundancy) if not self.online: self.watched = 0 utility.start_thread(acquaintance_watch_thread(self,node)) else: self.watched = 1
def startup_thread(self, status_monitor=status_monitor): list = self.acquaintances.values() list.sort(lambda x,y: cmp(x.sort_value(),y.sort_value())) for item in list: item.start_watching(self.node) #start watching tends to breed, try to make sure we don't get #too many threads. #yes, this is hacky #print item.nickname, threading.activeCount() #time.sleep(0.25) while 1: yield 'sleep',0.25 if threading.activeCount() < 40: break self.me.start_watching(self.node) while not self.me.watched: yield 'sleep',0.1 online = self.me.online address = self.me.address if online: if status_monitor: status_monitor(_('Shutting down your other peer.')) while 1: ticket,template,wait = self.node.call(address, ('identity abort',)) if wait: yield 'call',(self.node,ticket) try: dummy_result = self.node.get_reply(ticket, template) except error.Error: break yield 'sleep',4 self.me.online = 1 self.me.address = self.node.address self.me.connect_time = time.time() # Task to retrieve existing watchers # Task to poll existing watchers utility.start_thread(name_server_watch_poller_thread(self)) # now refresh my own offline presence pipe = self.node.retrieve(self.public_key_name_offline, settings.cache_redundancy) list = [ ] while not pipe.finished(): for item in pipe.read_all(): if type(item[1]) == types.DictType and \ item[1].get('type') == 'identity offline' and \ item[1].get('salt'): list.append(item) yield 'sleep',2 if not self.running: return pipe.stop() #if len(list) != 4: # print _("%d peers holding your offline presence.") % len(list) for item in list: address, value = item key = hash.hash_of(safe_pickle.dumps(self.sign(value['salt']))) ticket, template, wait = self.node.call(address, ('data cache remove',key)) if wait: yield 'call',(self.node,ticket) try: dummy_result = self.node.get_reply(ticket,template) except error.Error: pass self.lock.acquire() try: package = { 'name' : self.info['name'], 'human-name' : self.info['human-name'], 'description': self.info['description'], 'timezone' : self.info['timezone'], 'key' : self.public_key, 'keywords' : self.info['keywords'], } finally: self.lock.release() package_dumped = safe_pickle.dumps(package) signature = self.sign(package_dumped) # now publish and cache offline identity value = { 'type' : 'identity offline', 'package' : package_dumped, 'signature' : signature, 'salt' : utility.random_bytes(settings.name_bytes) } lock = hash.hash_of(hash.hash_of(safe_pickle.dumps(self.sign(value['salt'])))) publications = [ self.public_key_name_offline, self.service_name ] for item in package['keywords']: publications.append(hash.hash_of('identity-name '+item)) # thomasV # redundancy 4: this is the meta-publish result, publish_thread = self.app.cache.publish(publications,value,lock, 4) yield 'wait',publish_thread
def chat_tell_thread(chat, dests, message, attachment=None, augmentation=None): """Messages are sent in parallel, one thread per recipient. The information displayed in field ends with '...' until the last message has been sent, then with '..' until all recipients have answered, and finally with '.' The field is closed by the last thread to finish """ check.check_matches(dests, ['text']) check.check_is_text(message) std_timestamp = host2standard_timestamp(time.time()) if augmentation and len(augmentation) != len(message): sys.stderr.write("Augmentation %d bytes, should be %d bytes.\n" % (len(augmentation), len(message))) augmentation = None message_id = chat.create_message_id() try: result = ['','','','','...\n'] dest_list = [ ] dest_address = [ ] checked_address = [ ] dest_names = [ ] received_names = [ ] offline_list = [ ] offline_names = [ ] recipient_list = [ ] channel_pipes = { } chan_cnt=[0] for dest in dests: if dest[0]=='#': channel_pipes[dest] = chat.channels.sub_list_pipe(dest) recipient_list.append((None,None,dest)) continue acq = chat.app.name_server.locate(dest) acq.start_watching(chat.node) while not acq.watched: yield 'sleep',0.1 acq.lock.acquire() online = acq.online address = acq.address name = acq.name username = acq.info['name'] acq.lock.release() if not online: offline_list.append(acq) offline_names.append(dest) recipient_list.append((name,None,username)) else: dest_list.append(acq) dest_address.append(address) dest_names.append(dest) recipient_list.append((name,address,username)) my_details = ( chat.app.name_server.public_key_name, chat.app.name_server.name ) package = { 'from': my_details, 'to': recipient_list, 'text': message } if augmentation: package['aug'] = augmentation if attachment: package['attach'] = attachment package_pickle = safe_pickle.dumps(package) signature = chat.app.name_server.sign(package_pickle) message = ('chat message 2',package_pickle,signature) def recipient_thread(chat,address,name,channel,received_names,dest_address, checked_address,chan_cnt,message_id,result, message=message): succ = 1 try: ticket, template, wait = chat.node.call(address,message) if wait: yield ('call',(chat.node,ticket)) ret_value = chat.node.get_reply(ticket,template) #Type checking if type(ret_value) != types.TupleType: succ = 0 if not channel: result[2] = result[2] + name + _(' sent bad reply.\n') chat.update_message_status(message_id, result[0]+result[4]+result[2]) except error.Error,err: succ = 0 if not channel: result[2] = result[2] + name + _(' could not be contacted: ')+err.message+'\n' chat.update_message_status(message_id, result[0]+result[4]+result[2]) chat.app.name_server.bad_address(address) if succ: if channel: chan_cnt[0] += 1 else: received_names.append(name) if ret_value[0]: pass #if ret_value[1] != '': # result[2] = result[2] + name + ' ' + ret_value[1] + '\n' else: if ret_value[1] == '': result[2] = result[2] + name + _(' is quiet.\n') else: result[2] = result[2] + name + _(' is quiet: ') + ret_value[1] + '\n' if chan_cnt[0] == 0: result[0] = _('Message received by ') + utility.english_list(received_names) elif chan_cnt[0] == 1: result[0] = _('Message received by ')\ +utility.english_list(received_names+[ _('1 person')])+result[1] else: result[0] = _('Message received by ')\ +utility.english_list(received_names+[ _('%d people') % chan_cnt[0] ])+result[1] chat.update_message_status(message_id,result[0]+result[4]+result[2]) checked_address.append(address) checked_address.sort() if result[4]=='..\n' and checked_address == dest_address: if chan_cnt[0]==0 and received_names == []: result[0] = _('Nobody received your message') result[4] = '.\n' chat.update_message_status(message_id, result[0]+result[4]+result[2]) for i in range(len(dest_list)): utility.start_thread(recipient_thread(chat,dest_address[i],dest_names[i],0,\ received_names,dest_address,checked_address,chan_cnt,message_id,result)) if channel_pipes: if channel_pipes.__len__()>1: result[1] = (_(' on channels %s') % utility.english_list(channel_pipes.keys())) else: result[1] = (_(' on channel %s') % channel_pipes.keys()[0]) if channel_pipes: for chan_name in channel_pipes.keys(): if not chat.channels.cache.has_key(chan_name): chat.channels.cache[chan_name]=[] else: for address in chat.channels.cache[chan_name]: if address in dest_address: continue dest_address.append(address) utility.start_thread( recipient_thread(chat,address,'',1,received_names, dest_address,checked_address,chan_cnt,message_id,result)) #reset the cache: chat.channels.cache[chan_name]=[] while channel_pipes: for chan_name,chan_pipe in channel_pipes.items(): if chan_pipe.finished(): chan_pipe.stop() del channel_pipes[chan_name] continue for address in chan_pipe.read_all(): #update the cache if address not in chat.channels.cache[chan_name]: chat.channels.cache[chan_name].append(address) if address in dest_address: continue dest_address.append(address) utility.start_thread(recipient_thread(chat,address,'',1,received_names, dest_address,checked_address,chan_cnt,message_id,result)) yield 'sleep',0.1 #we now have launched all the tasks dest_address.sort() result[4]='..\n' if checked_address == dest_address: if chan_cnt[0]==0 and received_names == []: result[0] = _('Nobody received your message') result[4] = '.\n' chat.update_message_status(message_id, result[0]+result[4]+result[2]) recall_list = [ ] if offline_list: chat.update_message_status(message_id,'Caching message...\n',1) for i in range(len(offline_list)): key = utility.random_bytes(settings.name_bytes) lock = hash.hash_of(key) crypt = offline_list[i].encrypt((key, message, std_timestamp)) data = { 'type' : 'offline message', 'crypt': crypt } name = hash.hash_of('offline message '+offline_list[i].name) # Loose proof of @R50: @I21, @E22. recall_list.append((offline_list[i].nickname,name,data,key)) publish_result, subthread = chat.app.cache.publish([name],data,lock) yield 'wait',subthread if publish_result: redundancy = publish_result[0] else: redundancy = 0 if redundancy == settings.cache_redundancy: str = offline_names[i] + _(' will get your message when next logged on.\n') elif redundancy == 0: str = _('Could not store message for ') + offline_names[i] + '!\n' else: str = (offline_names[i] + _(' will probably get your message when next logged on:\n' +\ ' Message only stored with redundancy %d.\n') % redundancy) result[3] = result[3] + str chat.update_message_status(message_id,result[3]+'\n',1) chat.recall_list = recall_list
def retrieve_cached_messages(self,on_complete): utility.start_thread(retrieve_cached_messages_thread(self,on_complete))
def start(self): self.cache = utility.get_config("hash_cache",{ }) utility.Task_manager.start(self) self.node.add_handler('download chunk', self, ('name', 'integer', 'integer'), types.StringType) self.node.add_handler('files available', self) utility.start_thread(file_server_poll_thread(self))
def sync_gossip_thread(self): def gossip_fetch_thread(self,acq): self.fetch_threads+=1 if not self.running: self.fetch_threads-=1 return if acq.distance != None: #acq.start_watching(self.node, 1) acq.lock.acquire() online = acq.online address = acq.address distance = acq.distance acq.lock.release() if distance == None or not online: self.fetch_threads-=1 return pos = 0 fetch_timeout = node.make_timeout(settings.gossip_fetch_time) while not node.is_timed_out(fetch_timeout): try: ticket, template, wait = self.node.call(address,('gossip list',pos,pos+20)) if wait: yield 'call',(self.node,ticket) result = self.node.get_reply(ticket,template) if not check.matches(result, [(types.IntType, 'any', 'any')]): node.bad_peer(address, _("Bad reply to 'gossip list': ") + `result`) break if len(result) == 0: break all_ok = 1 # effic: sort self.gossip, sort the returned results, # to speed up searches for signatures. However, I # haven't yet seen gossip_fetch_task come up in # profiles. for item in result: already_there = 0 for wodge in self.gossip: if wodge.signature == item[2]: #if wodge.distance(self.app.name_server) > from_fixed(item[0])+distance: # # TODO: What to do with unit_decay_time? # wodge.initial_distance = from_fixed(item[0])+distance # wodge.initial_time = time.time() wodge.opinions[acq.name] = from_fixed(item[0]) - wodge.decay() already_there = 1 break if already_there: continue try: ticket, template, wait = self.node.call( address,('gossip get',item[2])) if wait: yield 'call',(self.node,ticket) string = self.node.get_reply(ticket, template) wodgewodge = safe_pickle.loads(string) except error.Error: all_ok = 0 break #TODO: Confirm signature of known people wodge = Wodge({ 'wodge': wodgewodge, 'string': string, 'signature': item[2], 'initial_time': time.time(), 'initial_distance': None, #'initial_distance': from_fixed(item[0]) + distance 'unit_decay_time': item[1], 'opinions': { acq.name : from_fixed(item[0]) }, 'collapsed': 0}) if not self.insert_wodge(wodge): all_ok = 0 break if not all_ok: break pos = pos + 18 except error.Error: break self.fetch_threads-=1 self.fetch_threads=0 for acq in self.app.name_server.acquaintances.values(): if acq.distance != None: #acq.start_watching(self.node, 1) #name server starts watching all #want start watching to be gradual, see name server start() while not acq.watched: yield 'sleep',1 if not self.running: return if acq.online: utility.start_thread(gossip_fetch_thread(self,acq)) while self.fetch_threads != 0: yield 'sleep',0.1 requests = self.update_requests self.update_requests = [ ] for item in requests: self.app.idle_add(item)
def handle(self, request, address,call_id): check.check_matches(request, (types.StringType,)) check.check_is_af_inet_address(address) #=@R30 if request[0] == 'identity test': check.check_matches(request[1:], ('name',)) return self.sign('identity test ' + request[1]) elif request[0] == 'identity query': return self.get_info() elif request[0] == 'identity watch': self.lock.acquire() try: if address not in self.watchers: self.watchers.append(address) status = self.status finally: self.lock.release() return status elif request[0] == 'identity abort': def abort_thread(self, address=address): id_test_result = [] yield 'wait', identity_test_thread(address,self.public_key,self.node,id_test_result) if not id_test_result[0]: print "error testing identity" return self.aborted = 1 self.app.shutdown(_('Reconnecting from different machine')) utility.start_thread(abort_thread(self)) return None elif request[0] == 'identity connecting': if not (len(request) >= 2): return error.Error('Name_server.handle: identity connecting: expecting tuple of at least 2.') self.acquire_lock('get acq') try: acq = self.acquaintances.get(request[1]) finally: self.release_lock('get acq') if acq is None: return error.Error(_("I don't know you.")) else: #if len(request) > 2: # status = request[2] #else: # status = { } def acq_change_thread(ns, acq, address, call_id, node): check.check_is_af_inet_address(address) #=@R29 try: result, subthread = try_address(acq,address,node) yield 'wait',subthread if not result[0]: return except error.Error: return #acq.lock.acquire() #acq.status = status #acq.lock.release() ns.acquaintance_status_changed(acq, 'connect') utility.start_thread(acq_change_thread(self, acq, address, call_id, self.node)) # Proof of @R29: @R30; deeply immutable by @E11. return None elif request[0] == 'identity status changed': if not (len(request) >= 3): return error.Error('Name_server.handle: identity status changed: expecting tuple of at least 3.') self.lock.acquire() try: acq = self.acquaintances.get(request[1],None) if acq: acq.lock.acquire() try: acq.status = request[2] finally: acq.lock.release() self.acquaintance_status_changed(acq, 'status changed') finally: self.lock.release() elif request[0] == 'identity disconnecting': if not (len(request) >= 2): return error.Error('Name_server.handle: identity disconnecting: expecting tuple of at least 2.') self.lock.acquire() if address in self.watchers: self.watchers.remove(address) if self.acquaintances.has_key(request[1]): acq = self.acquaintances[request[1]] self.lock.release() if acq.disconnection(address): self.acquaintance_status_changed(acq, 'disconnect', message=request[2]) return None else: self.lock.release() return error.Error(_("I don't know you."))
def request_update(self, callback): self.update_requests.append(callback) utility.start_thread(self.sync_gossip_thread())
def start(self, status_monitor=None): utility.Task_manager.start(self) acq_list = utility.get_checked_config('acquaintances', types.ListType, [ ]) for item in acq_list: if not check.matches(item, Acquaintance.map_tmpl): print _("Warning: corrupted acquaintances config file; ignoring item: "), item continue acq = Acquaintance(self, item, 1) self.acquaintances[acq.name] = acq self.nicknames[acq.nickname] = acq acq.start() def make_acquaintance_noaddr(self, info): name = key_name(info['key']) acq = Acquaintance(self, {'info': info, 'name': name, 'nickname': self.choose_nickname(info['name'])},0) self.acquaintances[name] = acq self.nicknames[acq.nickname] = acq acq.start() acq.start_watching(self.node) self.acquaintance_status_changed(acq, "create") return acq self.me = make_acquaintance_noaddr(self,self.info) # Other me may want to test identity # May start chatting before test complete self.node.add_handler('identity test', self, ('name',), crypto.pubkey.signature_tmpl) self.node.add_handler('identity query', self, (), Acquaintance.info_template) self.node.add_handler('identity watch', self, (), types.DictionaryType) self.node.add_handler('identity connecting', self) self.node.add_handler('identity status changed', self, ('any', Acquaintance.status_template)) self.node.add_handler('identity disconnecting', self,('string', 'opt-text')) self.node.add_handler('identity abort', self) self.node.publish(self.public_key_name,self.get_info_func, settings.identity_redundancy) self.node.publish(self.public_key_name_offline,self.get_info_func, settings.identity_redundancy) self.node.publish(self.service_name,self.get_info_func, settings.identity_redundancy) for item in self.info['keywords']: self.node.publish(hash.hash_of('identity-name '+item), self.get_info_func, settings.identity_redundancy) def startup_thread(self, status_monitor=status_monitor): list = self.acquaintances.values() list.sort(lambda x,y: cmp(x.sort_value(),y.sort_value())) for item in list: item.start_watching(self.node) #start watching tends to breed, try to make sure we don't get #too many threads. #yes, this is hacky #print item.nickname, threading.activeCount() #time.sleep(0.25) while 1: yield 'sleep',0.25 if threading.activeCount() < 40: break self.me.start_watching(self.node) while not self.me.watched: yield 'sleep',0.1 online = self.me.online address = self.me.address if online: if status_monitor: status_monitor(_('Shutting down your other peer.')) while 1: ticket,template,wait = self.node.call(address, ('identity abort',)) if wait: yield 'call',(self.node,ticket) try: dummy_result = self.node.get_reply(ticket, template) except error.Error: break yield 'sleep',4 self.me.online = 1 self.me.address = self.node.address self.me.connect_time = time.time() # Task to retrieve existing watchers # Task to poll existing watchers utility.start_thread(name_server_watch_poller_thread(self)) # now refresh my own offline presence pipe = self.node.retrieve(self.public_key_name_offline, settings.cache_redundancy) list = [ ] while not pipe.finished(): for item in pipe.read_all(): if type(item[1]) == types.DictType and \ item[1].get('type') == 'identity offline' and \ item[1].get('salt'): list.append(item) yield 'sleep',2 if not self.running: return pipe.stop() #if len(list) != 4: # print _("%d peers holding your offline presence.") % len(list) for item in list: address, value = item key = hash.hash_of(safe_pickle.dumps(self.sign(value['salt']))) ticket, template, wait = self.node.call(address, ('data cache remove',key)) if wait: yield 'call',(self.node,ticket) try: dummy_result = self.node.get_reply(ticket,template) except error.Error: pass self.lock.acquire() try: package = { 'name' : self.info['name'], 'human-name' : self.info['human-name'], 'description': self.info['description'], 'timezone' : self.info['timezone'], 'key' : self.public_key, 'keywords' : self.info['keywords'], } finally: self.lock.release() package_dumped = safe_pickle.dumps(package) signature = self.sign(package_dumped) # now publish and cache offline identity value = { 'type' : 'identity offline', 'package' : package_dumped, 'signature' : signature, 'salt' : utility.random_bytes(settings.name_bytes) } lock = hash.hash_of(hash.hash_of(safe_pickle.dumps(self.sign(value['salt'])))) publications = [ self.public_key_name_offline, self.service_name ] for item in package['keywords']: publications.append(hash.hash_of('identity-name '+item)) # thomasV # redundancy 4: this is the meta-publish result, publish_thread = self.app.cache.publish(publications,value,lock, 4) yield 'wait',publish_thread utility.start_thread(startup_thread(self))
result = self.node.get_reply(ticket, template) except error.Error,err: #print "error on disconnecting",err,address pass except: pass self.disconnect_threads-=1 # self.watchers might change while we are disconnecting (especially due to 127.0.0.1) # so we need to copy them before if not self.aborted: threads = [] for item in self.watchers: threads.append(id_disconnect_thread(self,item)) for thread in threads: utility.start_thread(thread) self.watchers = [ ] list = self.acquaintances.values() self.change_activity('calling acq.stop_watching') for item in list: def stopper_task(self, item): item.stop(self.node) utility.Task(stopper_task, self, item).start() #note that this might wait for all tasks: could be long utility.Task_manager.stop(self) self.change_activity('unpublishing self') self.node.unpublish(self.get_info_func) self.change_activity('removing handlers')