def read(self):
items = {}
for line in self.F.read().split(b'\n'):
if line:
key, value = line.split(b" ", 1)
items[to_str(key)] = to_str(value)
return items
def getHello(self, s):
msg = to_str(s.recv(1000))
print(('Hello msg: <%s>' % msg))
inx = None
hello, inx_txt = msg.split()
if hello == "HELLO":
inx = int(inx_txt)
return inx
def from_bytes(msg):
t, body = msg.split(b':', 1)
if t == CPusher.Type:
return CPusher.from_bytes(body)
elif t == CToken.Type:
return CToken.from_bytes(body)
elif t == CMessage.Type:
return CMessage.from_bytes(body)
else:
raise ValueError("Unknown CPacket type : '%s'" % (to_str(t), ))
def consume(self, inp):
#print(self, ".consume(): inp:", inp)
header_buffer = self.Buffer + inp
match = self.EOH_RE.search(header_buffer)
if not match:
self.Buffer = header_buffer
error = False
if len(header_buffer) > self.MAXREAD:
self.Error = "Request is too long: %d" % (len(header_buffer), )
error = True
return False, error, b''
i1, i2 = match.span()
self.Complete = True
self.Raw = header = header_buffer[:i1]
rest = header_buffer[i2:]
headers = {}
header = to_str(header)
lines = [l.strip() for l in header.split("\n")]
if lines:
self.Headline = headline = lines[0]
words = headline.split(" ", 2)
#print ("HTTPHeader: headline:", headline, " words:", words)
if len(words) != 3:
self.Error = "Can not parse headline. len(words)=%d" % (
len(words), )
return True, True, b'' # malformed headline
if words[0].lower().startswith("http/"):
self.StatusCode = int(words[1])
self.StatusMessage = words[2]
self.Protocol = words[0].upper()
else:
self.Method = words[0].upper()
self.Protocol = words[2].upper()
self.URI = self.OriginalURI = uri = words[1]
self.setURI(uri)
if '?' in uri:
# detach query part
self.Query = uri.split("?", 1)[1]
for l in lines[1:]:
if not l: continue
try:
h, b = tuple(l.split(':', 1))
headers[h.strip()] = b.strip()
except:
pass
self.Headers = headers
self.Buffer = b""
return True, False, rest
def processMsg(self, cmd, args, msg):
msg = to_str(msg)
#print 'processMsg(<%s>)' % msg
if not self.Username and cmd != 'HELLO':
ans = 'ERR Say HELLO first'
else:
try:
fcn = self.MsgDispatch[cmd]
except:
#print sys.exc_type, sys.exc_value
return None
ans = fcn(self, cmd, args, msg)
#print 'processMsg() -> %s' % ans
return ans
def doRead(self, fd, sel):
if fd != self.Sock.fileno() or not self.Enabled:
return
try:
msg, addr = self.Sock.recvfrom(10000)
except:
return # for stupid conncetion refused error in Linux
msg = to_str(msg)
#print ("CellListener.doRead: msg <%s> from %s" % (repr(msg), addr))
#if addr[0] == self.MyHost:
# return # do not talk to myself - bad sign
#print 'rcvd: <%s> from <%s>' % (msg, addr)
if not msg: return
words = msg.split()
if len(words) < 2: return
if words[1] != self.FarmName: return
cmd = words[0]
args = words[2:]
ans = None
#print ("CellListener.doRead: cmd: %s args: %s" % (cmd, args))
if cmd == 'ACCEPT':
ans = self.doAccept(args, msg, addr)
elif cmd == 'ACCEPTR':
ans = self.doAccept(args, msg, addr, nolocal=1)
elif cmd == 'PING':
ans = self.doPing(args, msg, addr)
#print 'ans=%s' % ans
elif cmd == 'SEND':
ans = self.doSend(args, msg, addr)
elif cmd == 'SENDR':
ans = self.doSend(args, msg, addr, nolocal=1)
elif cmd == 'STATPSA':
ans = self.doStatPsa(args, msg, addr)
elif cmd == 'DPATH':
ans = self.doDPath(args, msg, addr)
elif cmd == 'OPEN':
ans = self.doOpen(args, msg, addr)
if ans != None:
#print 'sending: <%s> to <%s>' % (ans, addr)
try:
self.Sock.sendto(to_bytes(ans), addr)
except:
pass
def open(self, mode, tmo=None):
msg = 'OPEN %s %s %s %s %s %s' % (self.Info.Path, self.Info.CTime,
self.MyAddr[0], self.MyAddr[1], mode)
self.Mode = mode
t0 = time.time()
done = 0
while tmo == None or time.time() < t0 + tmo:
self.Sock.sendto(to_bytes(msg), self.DFC.CAddr)
r, w, e = select.select([self.Sock], [], [], 2.0)
if r:
reply, addr = self.Sock.recvfrom(10000)
reply = to_str(reply)
words = reply.split()
if len(words) >= 1 and words[0] == 'OK':
self.DAddr = addr
done = 1
break
return done
def run(self):
while True:
if self.Enabled:
try:
msg, addr = self.Sock.recvfrom(10000)
except:
continue
if not msg: continue
msg = to_str(msg)
self.debug("run: msg: [%s]" % (msg, ))
#if addr[0] == self.MyHost:
# return # do not talk to myself - bad sign
#print 'rcvd: <%s> from <%s>' % (msg, addr)
words = msg.split()
if len(words) < 2: continue
if words[1] != self.FarmName: continue
cmd = words[0]
args = words[2:]
ans = None
#print("CellListener.run: cmd: %s args: %s" % (cmd, args))
if cmd == 'ACCEPT':
ans = self.doAccept(args, msg, addr, False)
elif cmd == 'ACCEPTR':
ans = self.doAccept(args, msg, addr, True)
elif cmd == 'PING':
ans = self.doPing(args, msg, addr)
#print 'ans=%s' % ans
elif cmd == 'SEND':
ans = self.doSend(args, msg, addr, False)
elif cmd == 'SENDR':
ans = self.doSend(args, msg, addr, True)
elif cmd == 'STATPSA':
ans = self.doStatPsa(args, msg, addr)
elif cmd == 'DPATH':
ans = self.doDPath(args, msg, addr)
elif cmd == 'OPEN':
ans = self.doOpen(args, msg, addr)
if ans != None:
try:
self.Sock.sendto(to_bytes(ans), addr)
except:
pass
else:
self.sleep()
def localDataPath(self, lpath, info):
sock = socket(AF_INET, SOCK_DGRAM)
r = []
retry = 5
while retry > 0 and not r:
msg = to_bytes('DPATH %s %s %s' %
(self.FarmName, lpath, info.CTime))
try:
sock.sendto(msg, ('127.0.0.1', self.CAddr[1]))
except:
break
r, w, e = select.select([sock], [], [], 3)
retry = retry - 1
ans = None
if r:
ans, addr = sock.recvfrom(10000)
if not ans:
ans = None
sock.close()
return to_str(ans)
def run(self):
sock = socket(AF_INET, SOCK_DGRAM)
sock.settimeout(self.Timeout)
pingmsg = to_bytes('PING %s' % self.FarmName)
nretries = self.Retries
nodes = set(self.AddrMap.keys())
t0 = {}
try:
while nodes and nretries > 0:
nretries -= 1
# send pings
for n in nodes:
addr = (self.AddrMap[n], self.Port)
sock.sendto(pingmsg, addr)
t0[n] = time.time()
# listen to the echo
timed_out = False
while not timed_out and nodes:
try:
msg, addr = sock.recvfrom(10000)
#print("Pinger: received:", msg)
except socket_timeout:
timed_out = True
else:
words = tuple(to_str(msg).split(None, 2))
#print(words)
if len(words) == 3 and words[0] == "PONG":
cid = words[1]
data = json.loads(words[2])
if cid in nodes:
dt = time.time() - t0.get(cid)
self.emit((cid, addr, dt, data))
nodes.remove(cid)
finally:
self.close()
sock.close()
def cellInfo(self, node):
sock = socket(AF_INET, SOCK_DGRAM)
sock.sendto(to_bytes('STATPSA %s' % self.FarmName),
(node, self.CAddr[1]))
r, w, e = select.select([sock], [], [], 30)
if not r:
sock.close()
return None
ans, addr = sock.recvfrom(100000)
ans = to_str(ans)
lines = ans.split('\n')
st = CellInfo(node)
# parse PSAs
psalst = []
while lines:
l = lines[0]
lines = lines[1:]
if l == '.': break
words = l.split()
if len(words) < 5: continue
psn, size, used, rsrvd, free = tuple(words[:5])
size = int(size)
used = int(used)
rsrvd = int(rsrvd)
free = int(free)
psalst.append((psn, size, used, rsrvd, free))
st.PSAs = psalst
# parse transactions
txlst = []
while lines:
l = lines[0]
lines = lines[1:]
if l == '.': break
words = l.split()
if len(words) < 3: continue
txlst.append(tuple(words[:3]))
st.Txns = txlst
return st
def format_x509_name(self, x509_name):
components = [(to_str(k), to_str(v))
for k, v in x509_name.get_components()]
return "/".join(f"{k}={v}" for k, v in components)
def scan(self, recursive, with_meta):
#print(f"scan({self.Location}, rec={recursive}, with_meta={with_meta}...")
server = self.Server
location = self.Location
timeout = self.Timeout
lscommand = "xrdfs %s ls %s %s %s" % (server, "-l" if with_meta else "", "-R" if recursive else "", location)
files = []
dirs = []
status = "OK"
reason = ""
try:
#print(f"lscommand: {lscommand}")
retcode, out, err = ShellCommand.execute(lscommand, timeout=timeout)
#print(f"retcode: {retcode}")
except RuntimeError:
status = "timeout"
else:
if retcode:
if "not a directory" in err.lower():
return "OK", "", [], [location]
status = "ls failed"
reason = "status code: %s, stderr: [%s]" % (retcode, err)
command = "xrdfs %s stat %s" % (server, location)
subp = subprocess.Popen(command, shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
subp_out, subp_err = subp.communicate()
subp_out = to_str(subp_out)
subp_err = to_str(subp_err)
retcode = subp.returncode
if retcode:
status = "stat failed"
reason = "status: %d" % (retcode,)
if subp_err: reason += " " + subp_err.strip()
else:
for line in subp_out.split("\n"):
line = line.strip()
if line.startswith("Flags:"):
if not ("IsDir" in line):
files = [(location, None)]
status = "OK"
reason = ""
break
else:
lines = [x.strip() for x in out.split("\n")]
for l in lines:
if not l: continue
tup = self.parse_scan_line(l, with_meta)
if not tup or not tup[-1].startswith(location):
status = "failed"
reason = "Invalid line in output: %s" % (l,)
break
is_file, size, path = tup
if path.endswith("/."):
continue
path if path.startswith(location) else location + "/" + path # ????
if is_file:
files.append((path, size))
else:
dirs.append((path, size))
if False:
print("return from scan():")
print(" dirs:")
for d in dirs[:5]:
print(" ", d)
print(" files:")
for f in files[:5]:
print(" ", f)
return status, reason, dirs, files
import sys
if len(sys.argv[1:]) < 3:
print(Usage)
sys.exit(2)
cmd, inp, out = sys.argv[1:]
if sys.argv[1] == "compress":
if inp == "-":
inp = sys.stdin
else:
inp = open(inp, "r")
out = open(out, "wb")
w = PathListWrite_gzip(out)
for line in inp.readlines():
path = line.strip()
if path:
w.write(path)
w.close()
else:
inp = open(inp, "rb")
if out == "-":
out = sys.stdout
else:
out = open(out, "w")
r = PathListRead_gzip(inp)
for path in r.paths():
out.write(to_str(path) + "\n")
out.close()