def doStartListening(self, arg): """ Action method. """ try: _, info = arg self.router_proto_host = (info.proto, info.host) except: try: s = config.conf().getString('services/proxy-transport/current-router').strip() _, router_proto, router_host = s.split(' ') self.router_proto_host = (router_proto, router_host) except: lg.exc() self.router_identity = identitycache.FromCache(self.router_idurl) config.conf().setString('services/proxy-transport/current-router', '%s %s %s' % ( self.router_idurl, self.router_proto_host[0], self.router_proto_host[1])) if ReadMyOriginalIdentitySource(): lg.warn('my original identity is not empty') else: config.conf().setData('services/proxy-transport/my-original-identity', my_id.getLocalIdentity().serialize()) self.request_service_packet_id = [] callback.insert_inbox_callback(0, self._on_inbox_packet_received) if _Debug: lg.out(2, 'proxy_receiver.doStartListening !!!!!!! router: %s at %s://%s' % ( self.router_idurl, self.router_proto_host[0], self.router_proto_host[1]))
def doShowGUI(self, arg): lg.out(2, "initializer.doShowGUI") if settings.NewWebGUI(): from web import control d = control.init() else: from web import webcontrol d = webcontrol.init() try: from system.tray_icon import USE_TRAY_ICON except: USE_TRAY_ICON = False lg.exc() if USE_TRAY_ICON: from system import tray_icon if settings.NewWebGUI(): # tray_icon.SetControlFunc(control.on_tray_icon_command) tray_icon.SetControlFunc(self._on_tray_icon_command) else: tray_icon.SetControlFunc(webcontrol.OnTrayIconCommand) if not settings.NewWebGUI(): webcontrol.ready() if self.flagGUI or not self.is_installed: if settings.NewWebGUI(): def _show_gui(wsgiport): reactor.callLater(0.1, control.show) d.addCallback(_show_gui) # reactor.callLater(0.1, control.show) else: d.addCallback(webcontrol.show)
def pipe(cmdargs): """ Execute a process in different way, create a Pipe to do read/write operations with child process. See ``lib.nonblocking`` module. """ lg.out(6, "child_process.pipe %s" % str(cmdargs)) try: if bpio.Windows(): import win32process p = nonblocking.Popen( cmdargs, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False, creationflags=win32process.CREATE_NO_WINDOW, ) else: p = nonblocking.Popen( cmdargs, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False, ) except: lg.out(1, "child_process.pipe ERROR executing: %s" + str(cmdargs)) lg.exc() return None return p
def detach(cmdargs): """ """ lg.out(2, "child_process.detach %s" % str(cmdargs)) try: if bpio.Windows(): import win32process p = nonblocking.Popen( cmdargs, shell=False, # stdin=subprocess.PIPE, # stdout=subprocess.PIPE, # stderr=subprocess.PIPE, universal_newlines=False, creationflags=win32process.CREATE_NO_WINDOW | win32process.DETACHED_PROCESS, close_fds=True, ) else: p = nonblocking.Popen( cmdargs, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False, close_fds=True, ) except: lg.out(1, "child_process.detach ERROR executing: %s" + str(cmdargs)) lg.exc() return None return p
def _done(x, filename): try: fin = open(filename, 'rb') src = fin.read() fin.close() except: if output_func: output_func('error opening downloaded starter file') result.errback(Exception('error opening downloaded starter file')) return local_filename = os.path.join(GetLocalDir(), settings.WindowsStarterFileName()) bpio.backup_and_remove(local_filename) try: os.rename(filename, local_filename) lg.out(4, 'os_windows_update.download_and_replace_starter file %s was updated' % local_filename) except: lg.out(1, 'os_windows_update.download_and_replace_starter ERROR can not rename %s to %s ' % (filename, local_filename)) lg.exc() result.errback(Exception('can not rename the file ' + filename)) return python27dll_path = os.path.join(GetLocalDir(), 'python27.dll') if not os.path.exists(python27dll_path): lg.out(4, 'os_windows_update.download_and_replace_starter file "python27.dll" not found download from "%s" repo' % repo) url = settings.DefaultRepoURL(repo) + 'python27.dll' d = net_misc.downloadHTTP(url, python27dll_path) d.addCallback(_done_python27_dll, filename) d.addErrback(_fail, filename) return result.callback(1)
def doStartService(self, arg): """ Action method. """ if not self.installed(): self.automat('service-not-installed') return depends_results = [] for depend_name in self.dependent_on(): depend_service = services().get(depend_name, None) if depend_service is None: depends_results.append((depend_name, 'not found')) continue if depend_service.state != 'ON': depends_results.append((depend_name, 'not started')) continue if len(depends_results) > 0: self.automat('service-depend-off', depends_results) return lg.out(2, '[%s] STARTING' % self.service_name) try: result = self.start() except: lg.exc() self.automat('service-failed', 'exception when starting') return if isinstance(result, Deferred): result.addCallback(lambda x: self.automat('service-started')) result.addErrback(lambda x: self.automat('service-failed', x)) return if result: self.automat('service-started') else: self.automat('service-failed', 'result is %r' % result)
def GenerateHashBase(self): """ This make a long string containing all needed fields of ``packet`` (without Signature). Just to be able to generate a hash of the whole packet . """ sep = "-" stufftosum = "" try: stufftosum += str(self.Command) stufftosum += sep stufftosum += self.OwnerID stufftosum += sep stufftosum += self.CreatorID stufftosum += sep stufftosum += str(self.PacketID) stufftosum += sep stufftosum += self.Date stufftosum += sep stufftosum += self.Payload stufftosum += sep stufftosum += self.RemoteID except Exception as exc: lg.exc() raise exc return stufftosum
def on_received_ack_packet(self, payload): inp = cStringIO.StringIO(payload) try: stream_id = struct.unpack('i', inp.read(4))[0] except: inp.close() lg.exc() # self.session.automat('shutdown') return if stream_id not in self.streams.keys(): inp.close() # if not self.receivedFiles.has_key(stream_id): # lg.warn('unknown stream_id=%d in ACK packet from %s' % ( # stream_id, self.session.peer_address)) # self.session.automat('shutdown') if stream_id in self.dead_streams: # print 'old ack', stream_id pass else: if _Debug: lg.warn('%s - what a stream ???' % stream_id) # self.session.automat('shutdown') return try: self.streams[stream_id].on_ack_received(inp) except: lg.exc() self.session.automat('shutdown') inp.close()
def _got_my_address(self, value, key): if not isinstance(value, dict): lg.warn('can not read my address') self.automat('dht-write-failed') return try: addr = value[dht_service.key_to_hash(key)].strip('\n').strip() except: if _Debug: lg.out( 4, 'udp_node._got_my_address ERROR wrong key in response: %r' % value) lg.exc() self.automat('dht-write-failed') return if addr != '%s:%d' % (self.my_address[0], self.my_address[1]): if _Debug: lg.out( 4, 'udp_node._got_my_address ERROR value not fit: %r' % value) self.automat('dht-write-failed') return self.automat('dht-write-success')
def doReadKey(self, arg): # keyfn = arg['keyfilename'] src = arg['keysrc'] lg.out(2, 'installer.doReadKey length=%s' % len(src)) # src = bpio.ReadBinaryFile(keyfn) if len(src) > 1024 * 10: self.doPrint(('file is too big for private key', 'red')) return try: lines = src.splitlines() idurl = lines[0].strip() keysrc = '\n'.join(lines[1:]) if idurl != nameurl.FilenameUrl(nameurl.UrlFilename(idurl)): idurl = '' keysrc = src except: lg.exc() idurl = '' keysrc = src if self.state not in self.output: self.output[self.state] = {'data': [('', 'black')]} self.output[self.state] = {'data': [('', 'black')]} self.output[self.state]['idurl'] = idurl self.output[self.state]['keysrc'] = keysrc if 'RECOVER' not in self.output: self.output['RECOVER'] = {'data': [('', 'black')]} if keysrc and idurl: self.output['RECOVER']['data'].append(('private key and IDURL was loaded', 'green')) elif not idurl and keysrc: self.output['RECOVER']['data'].append(('private key was loaded, provide correct IDURL now', 'blue'))
def doReadAndUnserialize(self, arg): """ Action method. """ self.status, self.bytes_received, self.error_message = arg # DO UNSERIALIZE HERE , no exceptions newpacket = gateway.inbox(self) if newpacket is None: if _Debug: lg.out( _DebugLevel, "<<< IN <<< !!!NONE!!! [%s] %s from %s %s" % (self.proto.upper().ljust(5), self.status.ljust(8), self.host, os.path.basename(self.filename)), ) # net_misc.ConnectionFailed(None, proto, 'receiveStatusReport %s' % host) try: fd, _ = tmpfile.make("other", ".inbox.error") data = bpio.ReadBinaryFile(self.filename) os.write(fd, "from %s:%s %s\n" % (self.proto, self.host, self.status)) os.write(fd, str(data)) os.close(fd) except: lg.exc() try: os.remove(self.filename) except: lg.exc() self.automat("unserialize-failed", None) return self.label += "_%s[%s]" % (newpacket.Command, newpacket.PacketID) self.automat("valid-inbox-packet", newpacket)
def doSetUp(self, arg): """ Action method. """ self.hostname = settings.getIdServerHost() if self.hostname == '': self.hostname = misc.readExternalIP() # bpio.ReadTextFile(settings.ExternalIPFilename()) if self.hostname == '': self.hostname = net_misc.getLocalIp() lg.out(4, 'id_server.doSetUp hostname=%s' % self.hostname) if not os.path.isdir(settings.IdentityServerDir()): os.makedirs(settings.IdentityServerDir()) lg.out(4, ' created a folder %s' % settings.IdentityServerDir()) root = WebRoot() root.putChild('', WebMainPage()) try: self.web_listener = reactor.listenTCP(self.web_port, server.Site(root)) lg.out(4, " have started web listener on port %d " % (self.web_port)) except: lg.out(4, "id_server.set_up ERROR exception trying to listen on port " + str(self.web_port)) lg.exc() try: self.tcp_listener = reactor.listenTCP(self.tcp_port, IdServerFactory()) lg.out(4, " identity server listen on TCP port %d started" % (self.tcp_port)) except: lg.out(4, "id_server.set_up ERROR exception trying to listen on port " + str(self.tcp_port)) lg.exc()
def doSerializeAndWrite(self, arg): """ Action method. """ # serialize and write packet on disk a_packet = self.outpacket if self.route: a_packet = self.route["packet"] try: fileno, self.filename = tmpfile.make("outbox") self.packetdata = a_packet.Serialize() os.write(fileno, self.packetdata) os.close(fileno) self.filesize = len(self.packetdata) if self.filesize < 1024 * 10: self.timeout = 10 elif self.filesize > 1024 * 1024: self.timeout = int(self.filesize / float(settings.SendingSpeedLimit())) else: self.timeout = 300 # self.timeout = min( # settings.SendTimeOut() * 3, # max(int(self.filesize/(settings.SendingSpeedLimit()/len(queue()))), # settings.SendTimeOut())) except: lg.exc() self.packetdata = None self.automat("write-error")
def doReportSuccess(self, arg): """ Action method. """ try: min_port = min(map(lambda addr: addr[1], self.stun_results.values())) max_port = max(map(lambda addr: addr[1], self.stun_results.values())) my_ip = self.stun_results.values()[0][0] if min_port == max_port: result = ('stun-success', 'non-symmetric', my_ip, min_port) else: result = ('stun-success', 'symmetric', my_ip, self.stun_results) self.my_address = (my_ip, min_port) except: lg.exc() result = ('stun-failed', None, None, []) self.my_address = None if self.my_address: bpio.WriteFile(settings.ExternalIPFilename(), self.my_address[0]) bpio.WriteFile(settings.ExternalUDPPortFilename(), str(self.my_address[1])) if _Debug: lg.out(_DebugLevel, 'stun_client.doReportSuccess based on %d nodes: %s' % ( len(self.stun_results), str(self.my_address))) if _Debug: lg.out(_DebugLevel + 10, ' %s' % str(result)) for cb in self.callbacks: cb(result[0], result[1], result[2], result[3]) self.callbacks = []
def data_received(self, payload): """ """ from transport.tcp import tcp_connection inp = cStringIO.StringIO(payload) try: file_id = struct.unpack('i', inp.read(4))[0] file_size = struct.unpack('i', inp.read(4))[0] except: inp.close() lg.exc() return inp_data = inp.read() inp.close() if file_id not in self.inboxFiles: if len(self.inboxFiles) >= 2 * MAX_SIMULTANEOUS_OUTGOING_FILES: # too many incoming files, seems remote guy is cheating - drop # that session! lg.warn('too many incoming files, close connection %s' % str(self.connection)) self.connection.automat('disconnect') return self.create_inbox_file(file_id, file_size) self.inboxFiles[file_id].input_data(inp_data) if self.inboxFiles[file_id].is_done(): self.send_data(tcp_connection.CMD_OK, struct.pack('i', file_id)) self.inbox_file_done(file_id, 'finished')
def next_time(self): lasttime = self.lasttime if lasttime == '': # let it be one year ago (we can schedule 1 month maximum) and one day lasttime = str(time.time() - 366 * 24 * 60 * 60) try: # turned off - return -1 if self.type in ['none', 'disabled']: return -1 # every N seconds elif self.type == 'continuously': return maths.shedule_continuously(lasttime, int(self.interval),) # every N hours, exactly when hour begins, minutes and seconds are 0 elif self.type == 'hourly': return maths.shedule_next_hourly(lasttime, int(self.interval),) # every N days, at given time elif self.type == 'daily': return maths.shedule_next_daily(lasttime, self.interval, self.daytime) # every N weeks, at given time and selected week days elif self.type == 'weekly': week_days = self.details.split(' ') week_day_numbers = [] week_day_names = list(calendar.day_name) for week_label in week_days: try: i = week_day_names.index(week_label) except: continue week_day_numbers.append(i) return maths.shedule_next_weekly(lasttime, self.interval, self.daytime, week_day_numbers) # monthly, at given time and day elif self.type == 'monthly': month_dates = self.details.split(' ') return maths.shedule_next_monthly(lasttime, self.interval, self.daytime, month_dates) # yearly, at given time and month, day, NOT DONE YET! elif self.type == 'yearly': months_labels = self.details.split(' ') months_numbers = [] months_names = list(calendar.month_name) for month_label in months_labels: try: i = months_names.index(month_label) except: continue months_numbers.append(i) return maths.shedule_next_monthly(lasttime, self.interval, self.daytime, months_numbers) else: lg.out(1, 'schedule.next_time ERROR wrong schedule type: ' + self.type) return None except: lg.exc() return None
def from_memory(self, name): """ Read the constants from memory and take needed matrix on hands. """ # lg.out(6, "eccmap.from_memory with " + name) maxdata = 0 maxparity = 0 data = GetEccMapData(name) self.ParityToData = [] for PS in data: oneset = [] for datanum in PS: try: oneset.append(datanum) if datanum > maxdata: maxdata = datanum except (TypeError, ValueError): lg.out(1, 'eccmap.from_memory ERROR') lg.exc() if oneset: self.ParityToData.append(oneset) maxparity += 1 self.paritysegments = maxparity self.datasegments = maxdata + 1 # we only do this type at the moment self.type = 0
def request(self, request, info): from logs import lg from p2p import p2p_service words = request.Payload.split(' ') try: mode = words[1][:10] except: lg.exc() return None if mode != 'join' and mode != 'write' and mode != 'read': lg.out( 8, "service_accountant.request DENIED, wrong mode provided : %s" % mode) return None from coins import accountant_node if not accountant_node.A(): lg.out( 8, "service_accountant.request DENIED, accountant_node() state machine not exist") return p2p_service.SendFail( request, "accountant_node service not started") # if accountant_node.A().state not in ['ACCOUNTANTS?', "READY", "VALID_COIN?", "WRITE_COIN!", ]: # lg.out(8, "service_accountant.request DENIED, accountant_node() state is : %s" % accountant_node.A().state) # return p2p_service.SendFail(request, "accountant_node service # currently unavailable") if mode == 'join': accountant_node.A('accountant-connected', request.OwnerID) return p2p_service.SendAck(request, 'accepted')
def doCheckOverride(self, arg): """ Action method. """ target = arg.CreatorID idsrc = arg.Payload try: new_ident = identity.identity(xmlsrc=idsrc) except: lg.out(_DebugLevel, 'payload: [%s]' % idsrc) lg.exc() return if not new_ident.isCorrect() or not new_ident.Valid(): lg.warn('incoming identity is not valid') return if not self._is_my_contacts_present_in_identity(new_ident): cur_contacts = [] try: cur_contacts = identity.identity( xmlsrc=identitycache.ReadOverriddenIdentityXMLSource(target) ).getContacts() except: lg.exc() return if _Debug: lg.out(_DebugLevel, 'proxy_router.doCheckOverride override identity for %s' % arg.CreatorID) lg.out(_DebugLevel, ' current override contacts is : %s' % cur_contacts) lg.out(_DebugLevel, ' new contacts is : %s' % new_ident.getContacts()) identitycache.OverrideIdentity(arg.CreatorID, idsrc) else: if _Debug: lg.out(_DebugLevel, 'proxy_router.doCheckOverride skip override, found my contacts in identity from %s' % arg.CreatorID) lg.out(_DebugLevel, ' known contacts is : %s' % new_ident.getContacts())
def _got_peer_incoming(self, value, key, position): if _Debug: lg.out( _DebugLevel, 'udp_connector._got_peer_incoming at position %d: %d' % (position, len( str(value)))) self.working_deferred = None incoming = None if not isinstance(value, dict): self.automat('dht-read-failed') return try: # incoming = value.values()[0] incoming = value[dht_service.key_to_hash(key)] except: lg.out(2, '%r' % value) lg.exc() self.automat('dht-read-failed') return try: incoming_peer_id, incoming_user_address, time_placed = incoming.split( ' ') incoming_user_address = incoming_user_address.split(':') incoming_user_address = ( incoming_user_address[0], int( incoming_user_address[1])) except: lg.out(2, '%r' % incoming) lg.exc() self.automat('dht-read-failed') return self.automat( 'dht-read-success', (incoming_peer_id, incoming_user_address))
def stun_finished(x, block_marker): global _UDPListener global _StunClient global _WorkingDefers global _IsWorking global _LastStunResult global _TimeoutTask if block_marker: block_marker('unblock') lg.out(6, 'stun.stunExternalIP.stun_finished: ' + str(x).replace('\n', '')) _LastStunResult = x try: if _IsWorking: _IsWorking = False for d in _WorkingDefers: if x == '0.0.0.0': d.callback(x) else: d.callback(x) _WorkingDefers = [] _IsWorking = False if _UDPListener is not None and close_listener is True: _UDPListener.stopListening() _UDPListener = None if _StunClient is not None and close_listener is True: del _StunClient _StunClient = None except: lg.exc()
def cancel(self, request, info): from p2p import p2p_service if not contactsdb.is_customer(request.OwnerID): lg.warn( "got packet from %s, but he is not a customer" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.cancel created a new space file') space_dict = accounting.read_customers_quotas() if request.OwnerID not in space_dict.keys(): lg.warn( "got packet from %s, but not found him in space dictionary" % request.OwnerID) return p2p_service.SendFail(request, 'not a customer') try: free_bytes = int(space_dict['free']) space_dict['free'] = free_bytes + int(space_dict[request.OwnerID]) except: lg.exc() return p2p_service.SendFail(request, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(request.OwnerID) contactsdb.update_customers(new_customers) contactsdb.save_customers() space_dict.pop(request.OwnerID) accounting.write_customers_quotas(space_dict) from supplier import local_tester reactor.callLater(0, local_tester.TestUpdateCustomers) return p2p_service.SendAck(request, 'accepted')
def finishedStun(self): """ Called when the process is finished. """ local = '0.0.0.0' ip = '0.0.0.0' port = '0' typ = 'unknown' alt = 'unknown' try: if self.externalAddress and self.localAddress and self.natType: local = str(self.localAddress) ip = str(self.externalAddress[0]) port = str(self.externalAddress[1]) typ = str(self.natType.name) alt = str(self._altStunAddress) except: lg.exc() lg.out(2, 'stun.IPStunProtocol.finishedStun local=%s external=%s altStun=%s NAT_type=%s' % ( local, ip + ':' + port, alt, typ)) if self.result is not None: if not self.result.called: if ip == '0.0.0.0': self.result.callback(ip) else: self.result.callback(ip) self.result = None
def request(self, request, info): from logs import lg from p2p import p2p_service from main import settings words = request.Payload.split(" ") try: mode = words[1][:10] except: lg.exc() return None if mode != "route" and mode != "listen": lg.out(8, "service_broadcasting.request DENIED, wrong mode provided : %s" % mode) return None if not settings.enableBroadcastRouting(): lg.out(8, "service_broadcasting.request DENIED, broadcast routing disabled") return p2p_service.SendFail(request, "broadcast routing disabled") from broadcast import broadcaster_node if not broadcaster_node.A(): lg.out(8, "service_broadcasting.request DENIED, broadcast routing disabled") return p2p_service.SendFail(request, "broadcast routing disabled") if broadcaster_node.A().state not in ["BROADCASTING", "OFFLINE", "BROADCASTERS?"]: lg.out(8, "service_broadcasting.request DENIED, current state is : %s" % broadcaster_node.A().state) return p2p_service.SendFail(request, "currently not broadcasting") if mode == "route": broadcaster_node.A("new-broadcaster-connected", request.OwnerID) lg.out(8, "service_broadcasting.request ACCEPTED, mode: %s" % words) return p2p_service.SendAck(request, "accepted") if mode == "listen": broadcaster_node.A().add_listener(request.OwnerID, " ".join(words[2:])) lg.out(8, "service_broadcasting.request ACCEPTED, mode: %s" % words[1]) return p2p_service.SendAck(request, "accepted") return None
def CanMakeProgress(self, DataSegs, ParitySegs): """ Another method to check if we can do some data reconstruction. Lists are 1 for Data and Parity, lists are [0,1,1,1,0...] 0 is don't have 1 is have. """ for paritynum in range(self.paritysegments): # foreach parity if ParitySegs[paritynum] == 1: # If parity is not missing (so we can use it) Parity = self.ParityToData[paritynum] missing = 0 # We will see how many of the datas are missing for DataNum in Parity: # look at all datas that went into this parity try: if DataSegs[DataNum] != 1: # if this data is missing missing += 1 # increase count of those missing in this parity except: lg.exc() return False # keep track of the last missing in case only one is missing if missing == 1: # if missing exactly 1 of datas in parity, we can fix the data, have work to do return True else: # we're missing this parity, can we rebuild it Parity = self.ParityToData[paritynum] missing = 0 # We will see how many of the datas are missing for DataNum in Parity: # look at all datas that went into this parity try: if DataSegs[DataNum] != 1: # if this data is missing missing += 1 # increase count of those missing in this parity except: lg.exc() return False # keep track of the last missing in case only one is missing if missing == 0: # if missing none of the data for this parity, we have work to do return True return False
def loadfromfile(self, fname): """ This is old method, I decide to move all constants into the Python code. """ # lg.out(10, "eccmap.loadfromfile with " + fname) if os.path.exists(fname): filename = fname else: filename = os.path.join("..", fname) maxdata = 0 maxparity = 0 s = ReadTextFile(filename) for PS in re.split("\]", s): good = PS.lstrip("\[\,") oneset = [] for datastring in good.split(","): try: datanum = int(datastring) oneset.append(datanum) if datanum > maxdata: maxdata = datanum except (TypeError, ValueError): lg.exc() if oneset: self.ParityToData.append(oneset) maxparity += 1 self.paritysegments = maxparity self.datasegments = maxdata + 1 self.type = 0 # we only do this type at the moment
def ReadIndex(inpt): """ Read index data base, ``input`` is a ``cStringIO.StringIO`` object which keeps the data. This is a simple text format, see ``p2p.backup_fs.Serialize()`` method. The first line keeps revision number. """ global _LoadingFlag if _LoadingFlag: return False _LoadingFlag = True try: new_revision = int(inpt.readline().rstrip('\n')) except: _LoadingFlag = False lg.exc() return False backup_fs.Clear() count = backup_fs.Unserialize(inpt) if _Debug: lg.out(_DebugLevel, 'backup_control.ReadIndex %d items loaded' % count) # local_site.update_backup_fs(backup_fs.ListAllBackupIDsSQL()) commit(new_revision) _LoadingFlag = False return True
def datagramReceived(self, datagram, address): global _LastDatagramReceivedTime _LastDatagramReceivedTime = time.time() inp = cStringIO.StringIO(datagram) try: # version = datagram[0] # command = datagram[1] # payload = datagram[2:] # payloadsz = len(payload) datagramsz = len(datagram) version = inp.read(1) command = inp.read(1) except: inp.close() lg.exc() return if version != self.SoftwareVersion: inp.close() lg.warn('- different software version: %s' % version) return if _Debug: lg.out(_DebugLevel, '<<< [%s] (%d bytes) from %s, total %d bytes received' % ( command, datagramsz, str(address), self.bytes_in)) # self.bytes_in += datagramsz handled = False try: if self.command_filter_callback: handled = self.command_filter_callback(command, datagram, inp, address) except: lg.exc() payload = inp.read() inp.close() if not handled: self.run_callbacks((command, payload), address) self.bytes_in += datagramsz
def request(self, request, info): from p2p import p2p_service words = request.Payload.split(' ') try: bytes_for_customer = int(words[1]) except: lg.exc() bytes_for_customer = None if not bytes_for_customer or bytes_for_customer < 0: lg.warn("wrong storage value : %s" % request.Payload) return p2p_service.SendFail(request, 'wrong storage value') current_customers = contactsdb.customers() if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.request created a new space file') space_dict = accounting.read_customers_quotas() try: free_bytes = int(space_dict['free']) except: lg.exc() return p2p_service.SendFail(request, 'broken space file') if (request.OwnerID not in current_customers and request.OwnerID in space_dict.keys()): lg.warn("broken space file") return p2p_service.SendFail(request, 'broken space file') if (request.OwnerID in current_customers and request.OwnerID not in space_dict.keys()): lg.warn("broken customers file") return p2p_service.SendFail(request, 'broken customers file') if request.OwnerID in current_customers: free_bytes += int(space_dict[request.OwnerID]) space_dict['free'] = free_bytes current_customers.remove(request.OwnerID) space_dict.pop(request.OwnerID) new_customer = False else: new_customer = True from supplier import local_tester if free_bytes <= bytes_for_customer: contactsdb.update_customers(current_customers) contactsdb.save_customers() accounting.write_customers_quotas(space_dict) reactor.callLater(0, local_tester.TestUpdateCustomers) if new_customer: lg.out( 8, " NEW CUSTOMER - DENIED !!!!!!!!!!! not enough space") else: lg.out( 8, " OLD CUSTOMER - DENIED !!!!!!!!!!! not enough space") return p2p_service.SendAck(request, 'deny') space_dict['free'] = free_bytes - bytes_for_customer current_customers.append(request.OwnerID) space_dict[request.OwnerID] = bytes_for_customer contactsdb.update_customers(current_customers) contactsdb.save_customers() accounting.write_customers_quotas(space_dict) reactor.callLater(0, local_tester.TestUpdateCustomers) if new_customer: lg.out(8, " NEW CUSTOMER ACCEPTED !!!!!!!!!!!!!!") else: lg.out(8, " OLD CUSTOMER ACCEPTED !!!!!!!!!!!!!!") return p2p_service.SendAck(request, 'accepted')
def installed(self): from logs import lg try: from transport.tcp import tcp_interface except: lg.exc() return False return True
def shortPath(path): """ Get absolute 'short' path in Unicode, converts to 8.3 windows filenames. """ path_ = os.path.abspath(path) if not Windows(): return strng.to_text(path_) if not os.path.exists(path_): if os.path.isdir(os.path.dirname(path_)): res = shortPath(os.path.dirname(path_)) return strng.to_text(os.path.join(res, os.path.basename(path_))) return strng.to_text(path_) try: import win32api # @UnresolvedImport spath = win32api.GetShortPathName(path_) return strng.to_text(spath) except: lg.exc() return strng.to_text(path_)
def make_key_value(self, data): try: key_a = data[self.role_a][self.field_a] key_b = data[self.role_b][self.field_b] version_1 = '{}:{}'.format(key_a, key_b) version_2 = '{}:{}'.format(key_b, key_a) out = set() out.add(self.transform_key(version_1)) out.add(self.transform_key(version_2)) return out, None except ( AttributeError, ValueError, KeyError, IndexError, ): return None except Exception: lg.exc()
def listRemovableDrivesWindows(): """ Return a list of "removable" drives under Windows. """ l = [] try: import win32file # @UnresolvedImport drivebits = win32file.GetLogicalDrives() for d in range(1, 26): mask = 1 << d if drivebits & mask: # here if the drive is at least there drname = '%c:\\' % chr(ord('A') + d) t = win32file.GetDriveType(drname) if t == win32file.DRIVE_REMOVABLE: l.append(drname) except: lg.exc() return l
def run(Tester): global _CurrentProcess # lg.out(8, 'localtester.run ' + str(Tester)) if bpio.isFrozen() and bpio.Windows(): commandpath = 'bptester.exe' cmdargs = [commandpath, Tester] else: commandpath = 'bptester.py' cmdargs = [sys.executable, commandpath, Tester] if not os.path.isfile(commandpath): lg.out(1, 'localtester.run ERROR %s not found' % commandpath) return None lg.out(14, 'localtester.run execute: %s' % cmdargs) try: if bpio.Windows(): import win32process _CurrentProcess = nonblocking.Popen( cmdargs, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False, creationflags=win32process.CREATE_NO_WINDOW, ) else: _CurrentProcess = nonblocking.Popen( cmdargs, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False, ) except: lg.out(1, 'localtester.run ERROR executing: %s' % str(cmdargs)) lg.exc() return None return _CurrentProcess
def extract_done(retcode, backupID, tarfilename, callback_method): lg.out(4, 'restore_monitor.extract_done %s tarfile: %s, result: %s' % (backupID, tarfilename, str(retcode))) global OnRestoreDoneFunc _WorkingBackupIDs.pop(backupID, None) _WorkingRestoreProgress.pop(backupID, None) # tmpfile.throw_out(tarfilename, 'file extracted') if OnRestoreDoneFunc is not None: OnRestoreDoneFunc(backupID, 'restore done') if callback_method: try: callback_method(backupID, 'restore done') except: lg.exc() return retcode
def init(): """ """ if _Debug: lg.out(_DebugLevel, 'driver.init') available_services_dir = os.path.join(bpio.getExecutableDir(), 'services') loaded = set() for filename in os.listdir(available_services_dir): if not filename.endswith('.py') and not filename.endswith('.pyo') and not filename.endswith('.pyc'): continue if not filename.startswith('service_'): continue name = str(filename[:filename.rfind('.')]) if name in loaded: continue if name in disabled_services(): if _Debug: lg.out(_DebugLevel, '%s is hard disabled' % name) continue try: py_mod = importlib.import_module('services.' + name) except: if _Debug: lg.out(_DebugLevel, '%s exception during module import' % name) lg.exc() continue try: services()[name] = py_mod.create_service() except: if _Debug: lg.out(_DebugLevel, '%s exception while creating service instance' % name) lg.exc() continue loaded.add(name) if not services()[name].enabled(): if _Debug: lg.out(_DebugLevel, '%s is switched off' % name) continue enabled_services().add(name) if _Debug: lg.out(_DebugLevel, '%s initialized' % name) build_order() config.conf().addCallback('services/', on_service_enabled_disabled)
def _new_idurl_exist(idsrc, new_idurl, pos): if _Debug: lg.out( _DebugLevel, 'id_rotator.doSelectNewIDServer._new_idurl_exist %r already with same name' % new_idurl) latest_revision = my_id.getLocalIdentity().getRevisionValue() try: existing_identity_with_same_name = identity.identity( xmlsrc=idsrc) if not existing_identity_with_same_name.isCorrect(): raise Exception( 'remote identity not correct at position %r' % pos) if not existing_identity_with_same_name.Valid(): raise Exception( 'remote identity not valid at position %r' % pos) except: lg.exc() if pos + 1 >= len(target_hosts): self.automat('no-id-servers-found') else: _ping_one_server(pos + 1) return if existing_identity_with_same_name.getPublicKey( ) == my_id.getLocalIdentity().getPublicKey(): if latest_revision <= existing_identity_with_same_name.getRevisionValue( ): self.new_revision = max( self.new_revision or -1, existing_identity_with_same_name.getRevisionValue() + 1) lg.info( 'found my own identity on "old" ID server and will re-use that source again: %r' % new_idurl) if new_idurl not in self.possible_sources: self.possible_sources.append(new_idurl) self.automat('found-new-id-source', new_idurl) return if pos + 1 >= len(target_hosts): self.automat('no-id-servers-found') else: _ping_one_server(pos + 1)
def state_changed(self, oldstate, newstate, event, arg): """ This method intended to catch the moment when automat's state were changed. """ if newstate in ['CONNECTED', 'DISCONNECTED', 'NO_SERVICE']: supplierPath = settings.SupplierPath( self.supplier_idurl, customer_idurl=self.customer_idurl) if not os.path.isdir(supplierPath): try: os.makedirs(supplierPath) except: lg.exc() return bpio.WriteTextFile( settings.SupplierServiceFilename( self.supplier_idurl, customer_idurl=self.customer_idurl), newstate, )
def query_json(jdata): """ Input keys: + method: 'get', 'get_many' or 'get_all' + index: 'id', 'idurl', 'creator', etc. + key: key to read single record from db (optional) + start: low key limit to search records in range + end: high key limit to search records in range + offset: pagination offset + limit: pagination size Returns tuple: (generator object or None, error message) """ if not db() or not db().opened: return None, 'database is closed' method = jdata.pop('method', None) if method not in [ 'get', 'get_many', 'get_all', ]: return None, 'unknown method' callmethod = globals().get(method) if not callmethod: return None, 'failed to call target method' index_name = jdata.pop('index', None) if index_name not in db().indexes_names: return None, 'unknown index' if 'with_doc' not in jdata: jdata['with_doc'] = True if 'with_storage' not in jdata: jdata['with_storage'] = True try: result = callmethod(index_name, **jdata) except: lg.exc() return None, 'exception raised during processing' if jdata['with_doc'] and index_name != 'id': return (_clean_doc(r['doc']) for r in result), None return result, None
def run(args_list, base_dir=None, callback=None): # TODO: Currently disabled return None global _CurrentProcess if _CurrentProcess is not None: lg.warn('only one process at once') return None if bpio.Windows(): cmdargs = ['upnpc-static.exe', ] elif bpio.Linux(): cmdargs = ['upnpc', ] elif bpio.Mac(): cmdargs = ['upnpc', ] else: return None if bpio.Windows(): # if we run windows binaries - upnpc-static.exe can be in the system sub folder if not os.path.isfile(cmdargs[0]): if os.path.isfile(os.path.join('system', cmdargs[0])): cmdargs[0] = os.path.join('system', cmdargs[0]) else: lg.warn('can not find executable file ' + cmdargs[0]) return None cmdargs += args_list if _Debug: lg.out(_DebugLevel, 'run_upnpc.run is going to execute: %s' % cmdargs) try: out_data, returncode = execute_in_shell(cmdargs) except: lg.exc() return None if _Debug: lg.out(_DebugLevel, ' %s finished with return code: %s' % (str(_CurrentProcess), str(returncode))) _CurrentProcess = None return out_data
def ReadTextFile(filename): """ Read text file and return its content. Also replace line endings: \r\n with \n - convert to Linux file format. """ if not os.path.isfile(filename): return '' if not os.access(filename, os.R_OK): return '' try: file = open(filename, "r") data = file.read() file.close() # Windows/Linux trouble with text files return data.replace('\r\n', '\n') except: lg.exc() return ''
def dispatch_snapshot(snap): handled = 0 if snap.model_name in listeners(): for listener_callback in listeners()[snap.model_name]: try: listener_callback(snap) except: lg.exc() continue handled += 1 if '*' in listeners(): for listener_callback in listeners()['*']: try: listener_callback(snap) except: lg.exc() continue handled += 1 return handled
def doSetUp(self, arg): """ Action method. """ self.hostname = settings.getIdServerHost() if self.hostname == '': self.hostname = strng.to_bin(misc.readExternalIP( )) # bpio.ReadTextFile(settings.ExternalIPFilename()) if self.hostname == '': self.hostname = net_misc.getLocalIp() lg.out(4, 'id_server.doSetUp hostname=%s' % self.hostname) if not os.path.isdir(settings.IdentityServerDir()): os.makedirs(settings.IdentityServerDir()) lg.out( 4, ' created a folder %s' % settings.IdentityServerDir()) root = WebRoot() root.putChild('', WebMainPage()) try: self.tcp_listener = reactor.listenTCP(self.tcp_port, IdServerFactory()) lg.out( 4, " identity server listen on TCP port %d started" % (self.tcp_port)) except: lg.out( 4, "id_server.set_up ERROR exception trying to listen on port " + str(self.tcp_port)) lg.exc() try: self.web_listener = reactor.listenTCP(self.web_port, server.Site(root)) lg.out( 4, " have started web server at http://%s:%d " % (self.hostname, self.web_port)) except: lg.out( 4, "id_server.set_up ERROR exception trying to listen on port " + str(self.web_port)) lg.exc()
def Event(request, info): """ """ if _Debug: try: e_json = serialization.BytesToDict(request.Payload, keys_to_text=True) e_json['event_id'] e_json['payload'] except: lg.exc() return lg.out( _DebugLevel, "p2p_service.Event %s from %s with %d bytes in json" % ( e_json['event_id'], info.sender_idurl, len(request.Payload), ))
def make_dir(name, extension='', prefix=''): """ """ global _TempDirPath global _FilesDict if _TempDirPath is None: init() if name not in list(_FilesDict.keys()): name = 'all' try: dirname = tempfile.mkdtemp(extension, prefix, subdir(name)) _FilesDict[name][dirname] = time.time() except: lg.out(1, 'tmpfile.make_dir ERROR creating folder in ' + name) lg.exc() return None if _Debug: lg.out(_DebugLevel, 'tmpfile.make_dir ' + dirname) return dirname
def run_begin_file_receiving_callbacks(pkt_in): """ """ global _BeginFileReceivingCallbacksList if _Debug: lg.out(_DebugLevel, 'callback.run_begin_file_receiving_callbacks for %s' % pkt_in) handled = False for cb in _BeginFileReceivingCallbacksList: try: if cb(pkt_in): handled = True except: lg.exc() if handled: if _Debug: lg.out(_DebugLevel, ' handled by %s' % cb) break return handled
def _get(self, entryPath): elemList = self._parseEntryPath(entryPath) fpath = os.path.join(self.configDir, *elemList) if os.path.isdir(fpath): out = [] for x in self._listdir(fpath): childPath = '/'.join(elemList + [x]) out.append(childPath) return out elif os.path.isfile(fpath): data = None try: f = file(fpath, 'rb') data = f.read() f.close() except (OSError, IOError): lg.exc('error reading from file: %s' % fpath) return data return None
def push_signed_message(producer_id, queue_id, data, creation_time=None): # TODO: to be continue try: signed_data = signed.Packet( Command=commands.Event(), OwnerID=producer_id, CreatorID=my_id.getIDURL(), PacketID=packetid.UniqueID(), Payload=serialization.DictToBytes(data, keys_to_text=True), RemoteID=queue_id, KeyID=producer_id, ) except: lg.exc() raise Exception('sign message failed') return write_message(producer_id, queue_id, data=signed_data.Serialize(), creation_time=creation_time)
def doUpdateMyAddress(self, *args, **kwargs): """ Action method. """ try: typ, new_ip, new_port = args[0] new_addr = (new_ip, new_port) except: lg.exc() return if _Debug: lg.out(4, 'udp_node.doUpdateMyAddress typ=[%s]' % typ) if self.my_address: lg.out( 4, ' old=%s new=%s' % (str(self.my_address), str(new_addr))) else: lg.out(4, ' new=%s' % str(new_addr)) self.my_address = new_addr
def receive(options): global _MyIDURL global _MyHost global _InternalPort global _Listener from transport.tcp import tcp_interface if _Debug: lg.out(_DebugLevel, 'tcp_node.receive %r' % options) if _Listener: lg.warn('listener already exist') tcp_interface.interface_receiving_started(_MyHost, options) return _Listener try: _MyIDURL = options['idurl'] _InternalPort = int(options['tcp_port']) except: _MyIDURL = None _InternalPort = None lg.exc() return None try: _Listener = reactor.listenTCP( _InternalPort, TCPFactory(None, keep_alive=True)) # @UndefinedVariable _MyHost = net_misc.pack_address( (options['host'].split(b':')[0], int(_InternalPort))) tcp_interface.interface_receiving_started(_MyHost, options) except CannotListenError as ex: lg.warn('port "%d" is busy' % _InternalPort) tcp_interface.interface_receiving_failed('port is busy') return None except Exception as ex: try: e = ex.getErrorMessage() except: e = str(ex) tcp_interface.interface_receiving_failed(e) lg.exc() return None return _Listener
def update_single_identity(index, idurl, idobj): lg.out(16, 'dbwrite.update_single_identity') from web.identityapp.models import Identity if idurl is None: try: Identity.objects.get(id=index).delete() lg.out(16, ' deleted an item at index %d' % index) except: lg.exc() else: try: ident = Identity.objects.get(id=index) lg.out(16, ' updated an item at index %d' % index) except: ident = Identity(id=index) lg.out(16, ' created a new item with index %d' % index) ident.idurl = idurl ident.src = idobj.serialize() ident.save()
def run_begin_file_sending_callbacks(result_defer, remote_idurl, proto, host, filename, description, pkt_out): """ """ global _BeginFileSendingCallbacksList if _Debug: lg.out(_DebugLevel, 'callback.run_begin_file_sending_callbacks %s to %s:%s at %s' % ( description, proto, host, remote_idurl)) handled = False for cb in _BeginFileSendingCallbacksList: try: if cb(result_defer, remote_idurl, proto, host, filename, description, pkt_out): handled = True except: lg.exc() if handled: if _Debug: lg.out(_DebugLevel, ' handled by %s' % cb) break return handled
def cancel(self, json_payload, newpacket, info): from twisted.internet import reactor from logs import lg from main import events from p2p import p2p_service from contacts import contactsdb from storage import accounting from services import driver # from userid import global_id customer_idurl = newpacket.OwnerID if not contactsdb.is_customer(customer_idurl): lg.warn("got packet from %s, but he is not a customer" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') if accounting.check_create_customers_quotas(): lg.out(6, 'service_supplier.cancel created a new space file') space_dict = accounting.read_customers_quotas() if customer_idurl not in list(space_dict.keys()): lg.warn( "got packet from %s, but not found him in space dictionary" % customer_idurl) return p2p_service.SendFail(newpacket, 'not a customer') try: free_bytes = int(space_dict['free']) space_dict['free'] = free_bytes + int(space_dict[customer_idurl]) except: lg.exc() return p2p_service.SendFail(newpacket, 'broken space file') new_customers = list(contactsdb.customers()) new_customers.remove(customer_idurl) contactsdb.update_customers(new_customers) contactsdb.remove_customer_meta_info(customer_idurl) contactsdb.save_customers() space_dict.pop(customer_idurl) accounting.write_customers_quotas(space_dict) from supplier import local_tester reactor.callLater(0, local_tester.TestUpdateCustomers) # if driver.is_on('service_supplier_relations'): # from dht import dht_relations # reactor.callLater(0, dht_relations.close_customer_supplier_relation, customer_idurl) lg.out(8, " OLD CUSTOMER: TERMINATED !!!!!!!!!!!!!!") events.send('existing-customer-terminated', dict(idurl=customer_idurl)) return p2p_service.SendAck(newpacket, 'accepted')
def run_outbox_callbacks(pkt_out): """ """ global _OutboxPacketCallbacksList if _Debug: lg.out(_DebugLevel, 'callback.run_outbox_callbacks for %s' % pkt_out) lg.out(_DebugLevel, ' %s' % _OutboxPacketCallbacksList) handled = False for cb in _OutboxPacketCallbacksList: try: if cb(pkt_out): handled = True except: lg.exc() if handled: if _Debug: lg.out(_DebugLevel, ' handled by %s' % cb) break return handled
def set_global_state(st): """ This method is called from State Machines when ``state`` is changed: global_state.set_global_state('P2P ' + newstate) So ``st`` is a string like: 'P2P CONNECTED'. ``_GlobalStateNotifyFunc`` can be used to keep track of changing program states. """ global _GlobalState global _GlobalStateNotifyFunc oldstate = _GlobalState _GlobalState = st # lg.out(6, (' ' * 40) + '{%s}->{%s}' % (oldstate, _GlobalState)) if _GlobalStateNotifyFunc is not None and oldstate != _GlobalState: try: _GlobalStateNotifyFunc(_GlobalState) except: lg.exc()
def ReadTextFile(filename): """ Read text file and return its content. """ global _PlatformInfo if _PlatformInfo is None: _PlatformInfo = platform.uname() if not os.path.isfile(filename): return u'' if not os.access(filename, os.R_OK): return u'' try: infile = open(filename, 'rt', encoding="utf-8") data = infile.read() infile.close() return strng.to_text(data) except: lg.exc() return u''
def doReportNicknameExist(self, arg): """ Action method. """ lg.out( 8, 'nickname_observer.doReportNicknameExist : (%s, %s)' % (self.key, arg)) if self.result_callback is not None: try: key_info = dht_service.split_key(self.key) nick = key_info['key'] index = key_info['index'] except: lg.exc() nick = self.nickname index = 0 # nik, num = self.key.split(':') # num = int(num) self.result_callback('exist', nick, index, arg)
def verify(self, signature, message, signature_as_digits=True): if signature_as_digits: signature_raw = number.long_to_bytes(int(strng.to_text(signature))) if signature[0:1] == b'0': signature_raw = b'\x00' + signature_raw if not strng.is_bin(signature_raw): raise ValueError('signature must be byte string') if not strng.is_bin(message): raise ValueError('message must be byte string') h = hashes.sha1(message, return_object=True) try: pkcs1_15.new(self.keyObject).verify(h, signature_raw) result = True except (ValueError, TypeError, ): if _Debug: from logs import lg lg.exc() result = False return result
def longPath(path): """ Get absolute 'long' path in Unicode, convert to full path, even if it was in 8.3 format. """ path_ = os.path.abspath(path) if not Windows(): if not isinstance(path_, unicode): return unicode(path_) return path_ if not os.path.exists(path_): return unicode(path_) try: import win32api lpath = win32api.GetLongPathName(path_) return unicode(lpath) except: lg.exc() return unicode(path_)
def rename_key(current_key_id, new_key_id, keys_folder=None): """ """ if not keys_folder: keys_folder = settings.KeyStoreDir() if current_key_id not in known_keys(): lg.warn('key %s is not found' % current_key_id) return False if key_obj(current_key_id).isPublic(): current_key_filepath = os.path.join(keys_folder, current_key_id + '.public') new_key_filepath = os.path.join(keys_folder, new_key_id + '.public') is_private = False else: current_key_filepath = os.path.join(keys_folder, current_key_id + '.private') new_key_filepath = os.path.join(keys_folder, new_key_id + '.private') is_private = True try: os.rename(current_key_filepath, new_key_filepath) except: lg.exc() return False key_object = known_keys().pop(current_key_id) known_keys()[new_key_id] = key_object gc.collect() if _Debug: lg.out( _DebugLevel, 'my_keys.rename_key key %s renamed to %s' % ( current_key_id, new_key_id, )) lg.out( _DebugLevel, ' file %s renamed to %s' % ( current_key_filepath, new_key_filepath, )) events.send('key-renamed', data=dict(old_key_id=current_key_id, new_key_id=new_key_id, is_private=is_private)) return True