def init_transfer(self, bcast_msg, info, tmo): bsock = socket(AF_INET, SOCK_DGRAM) bsock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) t0 = time.time() ctl_listen_sock = socket(AF_INET, SOCK_STREAM) ctl_listen_sock.bind(('', 0)) ctl_listen_sock.listen(0) ctl_listen_port = ctl_listen_sock.getsockname()[1] bcast = '%s %s %d' % (bcast_msg, self.MyHost, ctl_listen_port) if info is not None: bcast += " " + info.serialize() peer_ctl_sock = None ctl_listen_sock.settimeout(1.0) while peer_ctl_sock is None and (tmo is None or time.time() < t0 + tmo): bsock.sendto(to_bytes(bcast), self.BroadcastAddress) try: peer_ctl_sock, peer_ctl_addr = ctl_listen_sock.accept() except timeout: pass else: done = True ctl_listen_sock.close() bsock.close() if peer_ctl_sock is None: raise DCTimeOut() data_listen_sock = socket(AF_INET, SOCK_STREAM) data_listen_sock.bind((self.MyHost, 0)) data_listen_port = data_listen_sock.getsockname()[1] data_listen_sock.listen(0) ctl_str = SockStream(peer_ctl_sock) ctl_str.send('DATA %s %s' % (self.MyHost, data_listen_port)) data_listen_sock.settimeout(tmo) try: peer_data_sock, peer_data_addr = data_listen_sock.accept() except timeout: peer_ctl_sock.close() raise RuntimeError('Data connection accept() time out') finally: data_listen_sock.close() return peer_ctl_sock, peer_ctl_addr, peer_data_sock, peer_data_addr
def doRead(self, fd, sel): if fd != self.Sock.fileno() or not self.Enabled: return try: msg, addr = self.Sock.recvfrom(10000) except: return # for stupid conncetion refused error in Linux msg = to_str(msg) #print ("CellListener.doRead: msg <%s> from %s" % (repr(msg), addr)) #if addr[0] == self.MyHost: # return # do not talk to myself - bad sign #print 'rcvd: <%s> from <%s>' % (msg, addr) if not msg: return words = msg.split() if len(words) < 2: return if words[1] != self.FarmName: return cmd = words[0] args = words[2:] ans = None #print ("CellListener.doRead: cmd: %s args: %s" % (cmd, args)) if cmd == 'ACCEPT': ans = self.doAccept(args, msg, addr) elif cmd == 'ACCEPTR': ans = self.doAccept(args, msg, addr, nolocal=1) elif cmd == 'PING': ans = self.doPing(args, msg, addr) #print 'ans=%s' % ans elif cmd == 'SEND': ans = self.doSend(args, msg, addr) elif cmd == 'SENDR': ans = self.doSend(args, msg, addr, nolocal=1) elif cmd == 'STATPSA': ans = self.doStatPsa(args, msg, addr) elif cmd == 'DPATH': ans = self.doDPath(args, msg, addr) elif cmd == 'OPEN': ans = self.doOpen(args, msg, addr) if ans != None: #print 'sending: <%s> to <%s>' % (ans, addr) try: self.Sock.sendto(to_bytes(ans), addr) except: pass
def open(self, mode, tmo=None): msg = 'OPEN %s %s %s %s %s %s' % (self.Info.Path, self.Info.CTime, self.MyAddr[0], self.MyAddr[1], mode) self.Mode = mode t0 = time.time() done = 0 while tmo == None or time.time() < t0 + tmo: self.Sock.sendto(to_bytes(msg), self.DFC.CAddr) r, w, e = select.select([self.Sock], [], [], 2.0) if r: reply, addr = self.Sock.recvfrom(10000) reply = to_str(reply) words = reply.split() if len(words) >= 1 and words[0] == 'OK': self.DAddr = addr done = 1 break return done
def doPing(self, args, msg, addr): # PING <farm name> [<host> <port>] np = cellmgr_global.DataServer.putTxns() ng = cellmgr_global.DataServer.getTxns() retaddr = addr if len(args) >= 2: try: retaddr = (args[0], int(args[1])) except: return None ans = 'PONG %s %d %d %s' % (self.MyID, np, ng, cellmgr_global.CellStorage.status()) #print("sending pong:", ans) try: self.Sock.sendto(to_bytes(ans), retaddr) except: raise return None
def run(self): while True: if self.Enabled: try: msg, addr = self.Sock.recvfrom(10000) except: continue if not msg: continue msg = to_str(msg) self.debug("run: msg: [%s]" % (msg, )) #if addr[0] == self.MyHost: # return # do not talk to myself - bad sign #print 'rcvd: <%s> from <%s>' % (msg, addr) words = msg.split() if len(words) < 2: continue if words[1] != self.FarmName: continue cmd = words[0] args = words[2:] ans = None #print("CellListener.run: cmd: %s args: %s" % (cmd, args)) if cmd == 'ACCEPT': ans = self.doAccept(args, msg, addr, False) elif cmd == 'ACCEPTR': ans = self.doAccept(args, msg, addr, True) elif cmd == 'PING': ans = self.doPing(args, msg, addr) #print 'ans=%s' % ans elif cmd == 'SEND': ans = self.doSend(args, msg, addr, False) elif cmd == 'SENDR': ans = self.doSend(args, msg, addr, True) elif cmd == 'STATPSA': ans = self.doStatPsa(args, msg, addr) elif cmd == 'DPATH': ans = self.doDPath(args, msg, addr) elif cmd == 'OPEN': ans = self.doOpen(args, msg, addr) if ans != None: try: self.Sock.sendto(to_bytes(ans), addr) except: pass else: self.sleep()
def localDataPath(self, lpath, info): sock = socket(AF_INET, SOCK_DGRAM) r = [] retry = 5 while retry > 0 and not r: msg = to_bytes('DPATH %s %s %s' % (self.FarmName, lpath, info.CTime)) try: sock.sendto(msg, ('127.0.0.1', self.CAddr[1])) except: break r, w, e = select.select([sock], [], [], 3) retry = retry - 1 ans = None if r: ans, addr = sock.recvfrom(10000) if not ans: ans = None sock.close() return to_str(ans)
def doStatPsa(self, args, msg, addr): # STATPSA <farm name> [<host> <port>] psalst = cellmgr_global.CellStorage.listPSAs() str = '' for psn in psalst: size, used, rsrvd, free = \ cellmgr_global.CellStorage.getPSA(psn).status() str = str + '%s %d %d %d %d\n' % (psn, size, used, rsrvd, free) str = str + '.\n' str = str + cellmgr_global.DataServer.statTxns() retaddr = addr if len(args) >= 2: try: retaddr = (args[0], int(args[1])) except: return None try: self.Sock.sendto(to_bytes(str), retaddr) except: pass return None
def read(self, size, tmo=None): msg = 'READ %s %d' % (self.Offset, size) t0 = time.time() data = None while tmo == None or time.time() < t0 + tmo: try: self.Sock.sendto(to_bytes(msg), self.DAddr) except: break r, w, e = select.select([self.Sock], [], [], 2.0) if r: reply, addr = self.Sock.recvfrom(10000) words = reply.split(':', 1) if len(words) >= 1: off = int(words[0]) if off == self.Offset: data = words[1] self.Offset = self.Offset + len(data) break else: self.reopen(tmo) return data
def cellInfo(self, node): sock = socket(AF_INET, SOCK_DGRAM) sock.sendto(to_bytes('STATPSA %s' % self.FarmName), (node, self.CAddr[1])) r, w, e = select.select([sock], [], [], 30) if not r: sock.close() return None ans, addr = sock.recvfrom(100000) ans = to_str(ans) lines = ans.split('\n') st = CellInfo(node) # parse PSAs psalst = [] while lines: l = lines[0] lines = lines[1:] if l == '.': break words = l.split() if len(words) < 5: continue psn, size, used, rsrvd, free = tuple(words[:5]) size = int(size) used = int(used) rsrvd = int(rsrvd) free = int(free) psalst.append((psn, size, used, rsrvd, free)) st.PSAs = psalst # parse transactions txlst = [] while lines: l = lines[0] lines = lines[1:] if l == '.': break words = l.split() if len(words) < 3: continue txlst.append(tuple(words[:3])) st.Txns = txlst return st
def run(self): sock = socket(AF_INET, SOCK_DGRAM) sock.settimeout(self.Timeout) pingmsg = to_bytes('PING %s' % self.FarmName) nretries = self.Retries nodes = set(self.AddrMap.keys()) t0 = {} try: while nodes and nretries > 0: nretries -= 1 # send pings for n in nodes: addr = (self.AddrMap[n], self.Port) sock.sendto(pingmsg, addr) t0[n] = time.time() # listen to the echo timed_out = False while not timed_out and nodes: try: msg, addr = sock.recvfrom(10000) #print("Pinger: received:", msg) except socket_timeout: timed_out = True else: words = tuple(to_str(msg).split(None, 2)) #print(words) if len(words) == 3 and words[0] == "PONG": cid = words[1] data = json.loads(words[2]) if cid in nodes: dt = time.time() - t0.get(cid) self.emit((cid, addr, dt, data)) nodes.remove(cid) finally: self.close() sock.close()
def do_declare(config, server_url, args): opts, args = getopt.getopt(args, "n:", ["namespace="]) opts = dict(opts) if not args or args[0] == "help": print(Usage) sys.exit(2) url = server_url + "/data/declare" metadata = json.load(open(args[0], "r")) # parse to validate JSON params = [] namespace = opts.get("-n") or opts.get("--namespace") if namespace: params.append("namespace=%s" % (namespace, )) params.append("dataset=%s" % (args[1], )) url += "?" + "&".join(params) tl = TokenLib() token = tl.get(server_url) if not token: print("No valid token found. Please obtain new token") sys.exit(1) response = requests.post( url, data=to_bytes(json.dumps(metadata)), headers={"X-Authentication-Token": token.encode()}) status = response.status_code if status / 100 != 2: print("Error: ", status, "\n", response.text) sys.exit(1) body = response.text print(body)
def doPing(self, args, msg, addr): # PING <farm name> [<host> <port>] np = self.DataServer.putTxns() ng = self.DataServer.getTxns() nr = self.DataServer.repTxns() retaddr = addr if len(args) >= 2: try: retaddr = (args[0], int(args[1])) except: return None data = dict(status=self.CellStorage.status(), nput=np, nget=ng, nrep=nr, psa_stats=self.CellStorage.psa_stats()) ans = 'PONG %s %s' % (self.MyID, json.dumps(data)) try: self.debug("CellListener.doPing: replying to: %s" % (retaddr, )) self.Sock.sendto(to_bytes(ans), retaddr) except: raise return None
def addToBody(self, data): if PY3: data = to_bytes(data) #print ("addToBody:", data) self.Body.append(data)
def as_bytes(self, original=False): return to_bytes(self.as_text(original))
def save(self): data = [to_bytes("%s %s" % (k, v)) for k, v in self.Items.items()] self.F.seek(0) #print ("data:", data) self.F.write(b'\n'.join(data)) self.F.truncate()
def to_bytes(self): return CPacket.to_bytes( self, b'%s:%d:%d:%s' % (self.MID, self.Src, self.Dst, to_bytes(self.Body)))
def __init__(self, src=None, dst=None, body='', mid=None): self.Src = src self.Dst = dst self.Body = body self.Type = CMessage.Type self.MID = to_bytes(mid or uuid.uuid4().hex)
def process(self, handle): #self.debug("processRequest()") request = handle.Header request.setURI(self.rewrite_uri(request.URI)) self.debug("%s request: %s" % (handle.CID, request.Headline)) env = dict(REQUEST_METHOD=request.Method.upper(), PATH_INFO=request.Path, SCRIPT_NAME="", SERVER_PROTOCOL=request.Protocol, QUERY_STRING=request.Query) env["wsgi.url_scheme"] = "http" ssl_info = handle.SSLInfo if ssl_info != None: subject, issuer = self.x509_names(ssl_info) env["SSL_CLIENT_S_DN"] = subject env["SSL_CLIENT_I_DN"] = issuer env["wsgi.url_scheme"] = "https" if request.Headers.get("Expect") == "100-continue": self.CSock.sendall(b'HTTP/1.1 100 Continue\n\n') env["query_dict"] = self.parseQuery(request.Query) #print ("processRequest: env={}".format(env)) body_length = None for h, v in request.Headers.items(): h = h.lower() if h == "content-type": env["CONTENT_TYPE"] = v elif h == "host": words = v.split(":", 1) words.append("") # default port number env["HTTP_HOST"] = v env["SERVER_NAME"] = words[0] env["SERVER_PORT"] = words[1] elif h == "content-length": env["CONTENT_LENGTH"] = body_length = int(v) else: env["HTTP_%s" % (h.upper().replace("-", "_"), )] = v env["wsgi.input"] = BodyFile(handle.Body, handle.Sock, body_length) out = [] try: out = self.App(env, handle.start_response) except: self.error("%s %s" % (handle.CID, traceback.format_exc())) handle.start_response("500 Error", [("Content-Type", "text/plain")]) handle.OutBuffer = error = traceback.format_exc() self.logError(handle.CAddr, error) if handle.OutBuffer: # from start_response handle.Sock.sendall(to_bytes(handle.OutBuffer)) byte_count = 0 for line in out: line = to_bytes(line) try: handle.Sock.sendall(line) except Exception as e: self.logError(self.CAddr, "error sending body: %s" % (e, )) break byte_count += len(line) else: self.logRequest(handle, byte_count) handle.close() self.debug("%s done" % (handle.CID))
def part(nparts, path): if nparts <= 1: return 0 if PY3: path = to_bytes(path) #print("part(", nparts, path,"): adler:", adler32(path)) return adler32(path) % nparts
def write(self, path): self.G.write(to_bytes(path + "\n"))