def init_transfer(self, bcast_msg, info, tmo):
bsock = socket(AF_INET, SOCK_DGRAM)
bsock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
t0 = time.time()
ctl_listen_sock = socket(AF_INET, SOCK_STREAM)
ctl_listen_sock.bind(('', 0))
ctl_listen_sock.listen(0)
ctl_listen_port = ctl_listen_sock.getsockname()[1]
bcast = '%s %s %d' % (bcast_msg, self.MyHost, ctl_listen_port)
if info is not None:
bcast += " " + info.serialize()
peer_ctl_sock = None
ctl_listen_sock.settimeout(1.0)
while peer_ctl_sock is None and (tmo is None
or time.time() < t0 + tmo):
bsock.sendto(to_bytes(bcast), self.BroadcastAddress)
try:
peer_ctl_sock, peer_ctl_addr = ctl_listen_sock.accept()
except timeout:
pass
else:
done = True
ctl_listen_sock.close()
bsock.close()
if peer_ctl_sock is None:
raise DCTimeOut()
data_listen_sock = socket(AF_INET, SOCK_STREAM)
data_listen_sock.bind((self.MyHost, 0))
data_listen_port = data_listen_sock.getsockname()[1]
data_listen_sock.listen(0)
ctl_str = SockStream(peer_ctl_sock)
ctl_str.send('DATA %s %s' % (self.MyHost, data_listen_port))
data_listen_sock.settimeout(tmo)
try:
peer_data_sock, peer_data_addr = data_listen_sock.accept()
except timeout:
peer_ctl_sock.close()
raise RuntimeError('Data connection accept() time out')
finally:
data_listen_sock.close()
return peer_ctl_sock, peer_ctl_addr, peer_data_sock, peer_data_addr
def doRead(self, fd, sel):
if fd != self.Sock.fileno() or not self.Enabled:
return
try:
msg, addr = self.Sock.recvfrom(10000)
except:
return # for stupid conncetion refused error in Linux
msg = to_str(msg)
#print ("CellListener.doRead: msg <%s> from %s" % (repr(msg), addr))
#if addr[0] == self.MyHost:
# return # do not talk to myself - bad sign
#print 'rcvd: <%s> from <%s>' % (msg, addr)
if not msg: return
words = msg.split()
if len(words) < 2: return
if words[1] != self.FarmName: return
cmd = words[0]
args = words[2:]
ans = None
#print ("CellListener.doRead: cmd: %s args: %s" % (cmd, args))
if cmd == 'ACCEPT':
ans = self.doAccept(args, msg, addr)
elif cmd == 'ACCEPTR':
ans = self.doAccept(args, msg, addr, nolocal=1)
elif cmd == 'PING':
ans = self.doPing(args, msg, addr)
#print 'ans=%s' % ans
elif cmd == 'SEND':
ans = self.doSend(args, msg, addr)
elif cmd == 'SENDR':
ans = self.doSend(args, msg, addr, nolocal=1)
elif cmd == 'STATPSA':
ans = self.doStatPsa(args, msg, addr)
elif cmd == 'DPATH':
ans = self.doDPath(args, msg, addr)
elif cmd == 'OPEN':
ans = self.doOpen(args, msg, addr)
if ans != None:
#print 'sending: <%s> to <%s>' % (ans, addr)
try:
self.Sock.sendto(to_bytes(ans), addr)
except:
pass
def open(self, mode, tmo=None):
msg = 'OPEN %s %s %s %s %s %s' % (self.Info.Path, self.Info.CTime,
self.MyAddr[0], self.MyAddr[1], mode)
self.Mode = mode
t0 = time.time()
done = 0
while tmo == None or time.time() < t0 + tmo:
self.Sock.sendto(to_bytes(msg), self.DFC.CAddr)
r, w, e = select.select([self.Sock], [], [], 2.0)
if r:
reply, addr = self.Sock.recvfrom(10000)
reply = to_str(reply)
words = reply.split()
if len(words) >= 1 and words[0] == 'OK':
self.DAddr = addr
done = 1
break
return done
def doPing(self, args, msg, addr):
# PING <farm name> [<host> <port>]
np = cellmgr_global.DataServer.putTxns()
ng = cellmgr_global.DataServer.getTxns()
retaddr = addr
if len(args) >= 2:
try:
retaddr = (args[0], int(args[1]))
except:
return None
ans = 'PONG %s %d %d %s' % (self.MyID, np, ng,
cellmgr_global.CellStorage.status())
#print("sending pong:", ans)
try:
self.Sock.sendto(to_bytes(ans), retaddr)
except:
raise
return None
def run(self):
while True:
if self.Enabled:
try:
msg, addr = self.Sock.recvfrom(10000)
except:
continue
if not msg: continue
msg = to_str(msg)
self.debug("run: msg: [%s]" % (msg, ))
#if addr[0] == self.MyHost:
# return # do not talk to myself - bad sign
#print 'rcvd: <%s> from <%s>' % (msg, addr)
words = msg.split()
if len(words) < 2: continue
if words[1] != self.FarmName: continue
cmd = words[0]
args = words[2:]
ans = None
#print("CellListener.run: cmd: %s args: %s" % (cmd, args))
if cmd == 'ACCEPT':
ans = self.doAccept(args, msg, addr, False)
elif cmd == 'ACCEPTR':
ans = self.doAccept(args, msg, addr, True)
elif cmd == 'PING':
ans = self.doPing(args, msg, addr)
#print 'ans=%s' % ans
elif cmd == 'SEND':
ans = self.doSend(args, msg, addr, False)
elif cmd == 'SENDR':
ans = self.doSend(args, msg, addr, True)
elif cmd == 'STATPSA':
ans = self.doStatPsa(args, msg, addr)
elif cmd == 'DPATH':
ans = self.doDPath(args, msg, addr)
elif cmd == 'OPEN':
ans = self.doOpen(args, msg, addr)
if ans != None:
try:
self.Sock.sendto(to_bytes(ans), addr)
except:
pass
else:
self.sleep()
def localDataPath(self, lpath, info):
sock = socket(AF_INET, SOCK_DGRAM)
r = []
retry = 5
while retry > 0 and not r:
msg = to_bytes('DPATH %s %s %s' %
(self.FarmName, lpath, info.CTime))
try:
sock.sendto(msg, ('127.0.0.1', self.CAddr[1]))
except:
break
r, w, e = select.select([sock], [], [], 3)
retry = retry - 1
ans = None
if r:
ans, addr = sock.recvfrom(10000)
if not ans:
ans = None
sock.close()
return to_str(ans)
def doStatPsa(self, args, msg, addr):
# STATPSA <farm name> [<host> <port>]
psalst = cellmgr_global.CellStorage.listPSAs()
str = ''
for psn in psalst:
size, used, rsrvd, free = \
cellmgr_global.CellStorage.getPSA(psn).status()
str = str + '%s %d %d %d %d\n' % (psn, size, used, rsrvd, free)
str = str + '.\n'
str = str + cellmgr_global.DataServer.statTxns()
retaddr = addr
if len(args) >= 2:
try:
retaddr = (args[0], int(args[1]))
except:
return None
try:
self.Sock.sendto(to_bytes(str), retaddr)
except:
pass
return None
def read(self, size, tmo=None):
msg = 'READ %s %d' % (self.Offset, size)
t0 = time.time()
data = None
while tmo == None or time.time() < t0 + tmo:
try:
self.Sock.sendto(to_bytes(msg), self.DAddr)
except:
break
r, w, e = select.select([self.Sock], [], [], 2.0)
if r:
reply, addr = self.Sock.recvfrom(10000)
words = reply.split(':', 1)
if len(words) >= 1:
off = int(words[0])
if off == self.Offset:
data = words[1]
self.Offset = self.Offset + len(data)
break
else:
self.reopen(tmo)
return data
def cellInfo(self, node):
sock = socket(AF_INET, SOCK_DGRAM)
sock.sendto(to_bytes('STATPSA %s' % self.FarmName),
(node, self.CAddr[1]))
r, w, e = select.select([sock], [], [], 30)
if not r:
sock.close()
return None
ans, addr = sock.recvfrom(100000)
ans = to_str(ans)
lines = ans.split('\n')
st = CellInfo(node)
# parse PSAs
psalst = []
while lines:
l = lines[0]
lines = lines[1:]
if l == '.': break
words = l.split()
if len(words) < 5: continue
psn, size, used, rsrvd, free = tuple(words[:5])
size = int(size)
used = int(used)
rsrvd = int(rsrvd)
free = int(free)
psalst.append((psn, size, used, rsrvd, free))
st.PSAs = psalst
# parse transactions
txlst = []
while lines:
l = lines[0]
lines = lines[1:]
if l == '.': break
words = l.split()
if len(words) < 3: continue
txlst.append(tuple(words[:3]))
st.Txns = txlst
return st
def run(self):
sock = socket(AF_INET, SOCK_DGRAM)
sock.settimeout(self.Timeout)
pingmsg = to_bytes('PING %s' % self.FarmName)
nretries = self.Retries
nodes = set(self.AddrMap.keys())
t0 = {}
try:
while nodes and nretries > 0:
nretries -= 1
# send pings
for n in nodes:
addr = (self.AddrMap[n], self.Port)
sock.sendto(pingmsg, addr)
t0[n] = time.time()
# listen to the echo
timed_out = False
while not timed_out and nodes:
try:
msg, addr = sock.recvfrom(10000)
#print("Pinger: received:", msg)
except socket_timeout:
timed_out = True
else:
words = tuple(to_str(msg).split(None, 2))
#print(words)
if len(words) == 3 and words[0] == "PONG":
cid = words[1]
data = json.loads(words[2])
if cid in nodes:
dt = time.time() - t0.get(cid)
self.emit((cid, addr, dt, data))
nodes.remove(cid)
finally:
self.close()
sock.close()
def do_declare(config, server_url, args):
opts, args = getopt.getopt(args, "n:", ["namespace="])
opts = dict(opts)
if not args or args[0] == "help":
print(Usage)
sys.exit(2)
url = server_url + "/data/declare"
metadata = json.load(open(args[0], "r")) # parse to validate JSON
params = []
namespace = opts.get("-n") or opts.get("--namespace")
if namespace:
params.append("namespace=%s" % (namespace, ))
params.append("dataset=%s" % (args[1], ))
url += "?" + "&".join(params)
tl = TokenLib()
token = tl.get(server_url)
if not token:
print("No valid token found. Please obtain new token")
sys.exit(1)
response = requests.post(
url,
data=to_bytes(json.dumps(metadata)),
headers={"X-Authentication-Token": token.encode()})
status = response.status_code
if status / 100 != 2:
print("Error: ", status, "\n", response.text)
sys.exit(1)
body = response.text
print(body)
def doPing(self, args, msg, addr):
# PING <farm name> [<host> <port>]
np = self.DataServer.putTxns()
ng = self.DataServer.getTxns()
nr = self.DataServer.repTxns()
retaddr = addr
if len(args) >= 2:
try:
retaddr = (args[0], int(args[1]))
except:
return None
data = dict(status=self.CellStorage.status(),
nput=np,
nget=ng,
nrep=nr,
psa_stats=self.CellStorage.psa_stats())
ans = 'PONG %s %s' % (self.MyID, json.dumps(data))
try:
self.debug("CellListener.doPing: replying to: %s" % (retaddr, ))
self.Sock.sendto(to_bytes(ans), retaddr)
except:
raise
return None
def addToBody(self, data):
if PY3: data = to_bytes(data)
#print ("addToBody:", data)
self.Body.append(data)
def as_bytes(self, original=False):
return to_bytes(self.as_text(original))
def save(self):
data = [to_bytes("%s %s" % (k, v)) for k, v in self.Items.items()]
self.F.seek(0)
#print ("data:", data)
self.F.write(b'\n'.join(data))
self.F.truncate()
def to_bytes(self):
return CPacket.to_bytes(
self, b'%s:%d:%d:%s' %
(self.MID, self.Src, self.Dst, to_bytes(self.Body)))
def __init__(self, src=None, dst=None, body='', mid=None):
self.Src = src
self.Dst = dst
self.Body = body
self.Type = CMessage.Type
self.MID = to_bytes(mid or uuid.uuid4().hex)
def process(self, handle):
#self.debug("processRequest()")
request = handle.Header
request.setURI(self.rewrite_uri(request.URI))
self.debug("%s request: %s" % (handle.CID, request.Headline))
env = dict(REQUEST_METHOD=request.Method.upper(),
PATH_INFO=request.Path,
SCRIPT_NAME="",
SERVER_PROTOCOL=request.Protocol,
QUERY_STRING=request.Query)
env["wsgi.url_scheme"] = "http"
ssl_info = handle.SSLInfo
if ssl_info != None:
subject, issuer = self.x509_names(ssl_info)
env["SSL_CLIENT_S_DN"] = subject
env["SSL_CLIENT_I_DN"] = issuer
env["wsgi.url_scheme"] = "https"
if request.Headers.get("Expect") == "100-continue":
self.CSock.sendall(b'HTTP/1.1 100 Continue\n\n')
env["query_dict"] = self.parseQuery(request.Query)
#print ("processRequest: env={}".format(env))
body_length = None
for h, v in request.Headers.items():
h = h.lower()
if h == "content-type": env["CONTENT_TYPE"] = v
elif h == "host":
words = v.split(":", 1)
words.append("") # default port number
env["HTTP_HOST"] = v
env["SERVER_NAME"] = words[0]
env["SERVER_PORT"] = words[1]
elif h == "content-length":
env["CONTENT_LENGTH"] = body_length = int(v)
else:
env["HTTP_%s" % (h.upper().replace("-", "_"), )] = v
env["wsgi.input"] = BodyFile(handle.Body, handle.Sock, body_length)
out = []
try:
out = self.App(env, handle.start_response)
except:
self.error("%s %s" % (handle.CID, traceback.format_exc()))
handle.start_response("500 Error",
[("Content-Type", "text/plain")])
handle.OutBuffer = error = traceback.format_exc()
self.logError(handle.CAddr, error)
if handle.OutBuffer: # from start_response
handle.Sock.sendall(to_bytes(handle.OutBuffer))
byte_count = 0
for line in out:
line = to_bytes(line)
try:
handle.Sock.sendall(line)
except Exception as e:
self.logError(self.CAddr, "error sending body: %s" % (e, ))
break
byte_count += len(line)
else:
self.logRequest(handle, byte_count)
handle.close()
self.debug("%s done" % (handle.CID))
def part(nparts, path):
if nparts <= 1: return 0
if PY3: path = to_bytes(path)
#print("part(", nparts, path,"): adler:", adler32(path))
return adler32(path) % nparts
def write(self, path):
self.G.write(to_bytes(path + "\n"))