def run_multiquery_wperf(self, destdir, nquery, interval = 5):
# define output file names
resfiles = ["{0}/{1}_{2}.res".format(destdir, self.fprefix, i) for i in range(nquery)]
timefile = "{0}/{1}.time".format(destdir, self.fprefix)
if self.profflg:
cpupath = "{0}/{1}.cpu".format(destdir, self.fprefix)
iopath = "{0}/{1}.io".format(destdir, self.fprefix)
cmds = [[self.pgbindir + "psql", "-d", db, "-Ukeisuke", "-c", self.query]
for db in self.dbs[:nquery]]
perfpath = "{0}/{1}.perf".format(destdir, self.fprefix)
perfcmd = self.perfcmd.format(perfpath) + "sleep {0}".format(interval)
perfcmd = shlex.split(perfcmd)
count = 0
elapsed = []
while count < 3:
self.restart_db(destdir)
self.output_queryplan(destdir)
sys.stdout.write("query started\n")
if self.iotraceflg:
pbt = sp.Popen(["blktrace", "/dev/md0", "-D", destdir])
if self.profflg:
# run mpstat and iostat
pcpu = sp.Popen(["mpstat", "-P", "ALL", "1"], stdout = open(cpupath, "w"))
pio = sp.Popen(["iostat", "-x", "1"], stdout = open(iopath, "w"))
try:
# run query
fos = [open(f, "w") for f in resfiles]
pid = os.fork()
if pid == 0:
while True: sp.call(perfcmd)
# t = 0
# perfpath = "{0}/{1}_{{0}}.perf".format(destdir, self.fprefix)
# perfcmd = self.perfcmd + "sleep {0}".format(interval)
# while True:
# sp.call(shlex.split(perfcmd.format(perfpath.format(t))))
# t += interval
else:
stime = monotonic_time()
procs = [sp.Popen(cmd, stdout = fo) for cmd, fo in zip(cmds, fos)]
rcs = [p.wait() for p in procs]
ftime = monotonic_time()
os.kill(pid, 9)
sys.stdout.write("query finished\n")
elapsed = ftime - stime
with open(timefile, "w") as ft:
ft.write("{0}\n".format(elapsed))
if [0 for rc in rcs] == rcs: break
else:
count += 1
sys.stderr.write("Query Execution Error : Errorcodes{0}\n".format(rcs))
time.sleep(5)
finally:
for fo in fos: fo.close()
if self.profflg:
pio.kill()
pcpu.kill()
if self.iotraceflg:
pbt.kill()
return elapsed
def speed_check_info(remote_file, test_seconds, block_size, output_time=0):
start = monotonic.monotonic_time()
bytes = 0
previous = start
last_output = start
f = remote_file
while previous < (start + test_seconds):
temp = f.read(block_size)
if len(temp)==0: #We've gotten all the file
break
bytes += block_size
current = monotonic.monotonic_time()
if (current - last_output) > output_time and output_time != 0:
print str(bytes/(current-start)) + " KB/s"
last_output = current
previous = current
finish = previous
elapsed = finish-start
average = bytes / elapsed
return average, bytes, elapsed, block_size
def run_query_wperf(self, destdir, interval = 1):
outputprefix = os.path.join(destdir, self.fprefix)
resfile = outputprefix + ".res"
timefile = outputprefix + ".time"
ioout = outputprefix + ".io"
cpuout = outputprefix + ".cpu"
perfout = outputprefix + ".perf"
cmd = [self.pgbindir + "psql", "-d", self.db, "-Ukeisuke", "-c", self.query]
perfcmd = self.perfcmd.format(perfout) + "sleep {0}".format(interval)
perfcmd = shlex.split(perfcmd)
count = 0
elapsed = []
while count < 3:
self.restart_db(destdir)
self.output_queryplan(destdir)
sys.stdout.write("query started\n")
if self.iotraceflg:
pbt = sp.Popen(["blktrace", "/dev/md0", "-D", destdir])
if self.profflg:
# run mpstat and iostat
pcpu = sp.Popen(["mpstat", "-P", "ALL", "1"], stdout = open(cpuout, "w"))
pio = sp.Popen(["iostat", "-x", "1"], stdout = open(ioout, "w"))
try:
# run query
fo = open(resfile, "w")
pid = os.fork()
if pid == 0:
while True: sp.call(perfcmd)
# t = 0
# perfout = "{0}/{1}_{{0}}.perf".format(destdir, self.fprefix)
# perfcmd = self.perfcmd + "sleep {0}".format(interval)
# while True:
# sp.call(shlex.split(perfcmd.format(perfout.format(t))))
# t += interval
else:
stime = monotonic_time()
rcode = sp.call(cmd, stdout = fo)
ftime = monotonic_time()
finally:
os.kill(pid, 9)
fo.close()
if self.profflg:
pio.kill()
pcpu.kill()
if self.iotraceflg:
pbt.kill()
sys.stdout.write("query finished\n")
elapsed = ftime - stime
with open(timefile, "w") as ft:
ft.write("{0}\n".format(elapsed))
if not rcode: break
else:
count += 1
sys.stderr.write("Query Execution Error : {0}\n".format(rcode))
time.sleep(5)
return elapsed
def run_multiquery_wperf(self, destdir, nquery):
self.cpus = range(min(nquery, 32))
outputprefix = os.path.join(destdir, self.fprefix)
resfiles = [outputprefix + "_{0}.res".format(i) for i in range(nquery)]
timefile = outputprefix + ".time"
perfout = outputprefix + ".perf"
ioout = outputprefix + ".io"
cpuout = outputprefix + ".cpu"
cmds = [[self.pgbindir + "psql", "-d", db, "-Ukeisuke", "-c", self.query]
for db in self.dbs[:nquery]]
perfcmd = ["perf", "stat", "--all-cpus", "--no-aggr",
"--output", perfout, "--append",
"--event=" + ','.join([d["select"] for d in self.perfevents]),
"sleep", str(self.interval)]
count, elapsed = 0, 0
while count < 3:
self.restart_db(destdir)
self.output_queryplan(self.dbs[0], destdir)
sys.stdout.write("query started\n")
if self.iotraceflg: pbt = sp.Popen(["blktrace", "/dev/md0", "-D", destdir])
if self.profflg:
# run mpstat and iostat
pcpu = sp.Popen(["mpstat", "-P", "ALL", "1"], stdout = open(cpuout, "w"))
pio = sp.Popen(["iostat", "-x", "1"], stdout = open(ioout, "w"))
fos = [open(f, "w") for f in resfiles]
try:
# run query
pid = os.fork()
if pid == 0:
while True: sp.call(perfcmd)
else:
stime = monotonic_time()
procs = [sp.Popen(cmd, stdout = fo) for cmd, fo in zip(cmds, fos)]
rcs = [p.wait() for p in procs]
ftime = monotonic_time()
os.kill(pid, 9)
sys.stdout.write("query finished\n")
elapsed = ftime - stime
with open(timefile, "w") as ft:
ft.write("{0}\n".format(elapsed))
if [0 for rc in rcs] == rcs: break
else:
count += 1
sys.stderr.write("Query Execution Error : Errorcodes{0}\n".format(rcs))
time.sleep(5)
finally:
for fo in fos: fo.close()
if self.profflg:
pio.kill()
pcpu.kill()
if self.iotraceflg: pbt.kill()
return elapsed
def push_block_size(remote_file,
block_size,
block_window=1.0,
telemetry=False):
""" Grow the blocksize to a suitable value for this connection
If the block is downloaded in less than block_window seconds, then double it.
Repeat until we have two runs in a row that take longer than block_window
"""
f = remote_file
#prime the pump
temp = f.read(block_size)
temp = f.read(block_size)
temp = f.read(block_size)
start = monotonic.monotonic_time()
previous = start
last = False
second = False
debug_data = []
while not last and not second:
temp = f.read(block_size)
if len(temp) == 0: #We've gotten all the file
raise IOError("file was to short to complete setup")
current = monotonic.monotonic_time()
debug_data.append((current - previous, block_size))
if (current - previous) * 32 < block_window:
multiplier = block_window / (current - previous)
multiplier = multiplier / 16 # add in some leeway
block_size = int(block_size * multiplier)
if telemetry:
print "increasing block size to: " + str(block_size) + " B"
last = False
second = False
elif (current - previous) < block_window:
block_size = block_size * 2
if telemetry:
print "doubling block size to: " + str(block_size) + " B"
last = False
second = False
elif last:
#TODO: if elapsed time is more than a few percent over block_window, lower block_size
second = True
else:
#TODO: if elapsed time is more than a few percent over block_window, lower block_size
last = True
previous = current
debug_data.append((current - previous, block_size))
for t, b in debug_data:
print t, ",", b
return block_size
def push_block_size(remote_file, block_size, block_window=1.0, telemetry=False):
""" Grow the blocksize to a suitable value for this connection
If the block is downloaded in less than block_window seconds, then double it.
Repeat until we have two runs in a row that take longer than block_window
"""
f = remote_file
#prime the pump
temp = f.read(block_size)
temp = f.read(block_size)
temp = f.read(block_size)
start = monotonic.monotonic_time()
previous = start
last = False
second = False
debug_data = []
while not last and not second:
temp = f.read(block_size)
if len(temp)==0: #We've gotten all the file
raise IOError("file was to short to complete setup")
current = monotonic.monotonic_time()
debug_data.append((current-previous, block_size))
if(current-previous)*32 < block_window:
multiplier = block_window / (current-previous)
multiplier = multiplier / 16 # add in some leeway
block_size = int(block_size * multiplier)
if telemetry:
print "increasing block size to: " + str(block_size) + " B"
last = False
second = False
elif(current-previous) < block_window:
block_size = block_size * 2
if telemetry:
print "doubling block size to: " + str(block_size) + " B"
last = False
second = False
elif last:
#TODO: if elapsed time is more than a few percent over block_window, lower block_size
second = True
else:
#TODO: if elapsed time is more than a few percent over block_window, lower block_size
last = True
previous = current
debug_data.append((current-previous, block_size))
for t,b in debug_data:
# print t,",",b
pass
return block_size
def run_query_wperf(self, destdir):
outputprefix = os.path.join(destdir, self.fprefix)
resfile = outputprefix + ".res"
timefile = outputprefix + ".time"
ioout = outputprefix + ".io"
cpuout = outputprefix + ".cpu"
perfout = outputprefix + ".perf"
cmd = [self.pgbindir + "psql", "-d", self.db, "-Ukeisuke", "-c", self.query]
perfcmd = ["perf", "stat", "--all-cpus", "--no-aggr",
"--output", perfout, "--append",
"--event=" + ','.join([d["select"] for d in self.perfevents]),
"sleep", str(self.interval)]
count, elapsed = 0, 0
while count < 3:
self.restart_db(destdir)
self.output_queryplan(self.db, destdir)
sys.stdout.write("query started\n")
if self.iotraceflg: pbt = sp.Popen(["blktrace", "/dev/md0", "-D", destdir])
if self.profflg:
# run mpstat and iostat
pcpu = sp.Popen(["mpstat", "-P", "ALL", "1"], stdout = open(cpuout, "w"))
pio = sp.Popen(["iostat", "-x", "1"], stdout = open(ioout, "w"))
fo = open(resfile, "w")
pid = os.fork()
# run query
if pid == 0:
while True: sp.call(perfcmd)
else:
try:
stime = monotonic_time()
rcode = sp.call(cmd, stdout = fo)
ftime = monotonic_time()
finally:
os.kill(pid, 9)
fo.close()
if self.profflg:
pio.kill()
pcpu.kill()
if self.iotraceflg: pbt.kill()
sys.stdout.write("query finished\n")
elapsed = ftime - stime
with open(timefile, "w") as ft:
ft.write("{0}\n".format(elapsed))
if not rcode: break
else:
count += 1
sys.stderr.write("Query Execution Error : {0}\n".format(rcode))
time.sleep(5)
return elapsed
def run_query(self, destdir):
outputprefix = os.path.join(destdir, self.fprefix)
resfile = outputprefix + ".res"
timefile = outputprefix + ".time"
ioout = outputprefix + ".io"
cpuout = outputprefix + ".cpu"
perfout = outputprefix + ".perf"
cmd = [self.pgbindir + "psql", "-d", self.db, "-Ukeisuke", "-c", self.query]
count, elapsed = 0, 0
while count < 3:
self.restart_db(destdir)
self.output_queryplan(self.db, destdir)
sys.stdout.write("query started\n")
if self.iotraceflg: pbt = sp.Popen(["blktrace", "/dev/md0", "-D", destdir])
if self.profflg:
# run mpstat and iostat
pcpu = sp.Popen(["mpstat", "-P", "ALL", "1"], stdout = open(cpuout, "w"))
pio = sp.Popen(["iostat", "-x", "1"], stdout = open(ioout, "w"))
fo = open(resfile, "w")
try:
# run query
stime = monotonic_time()
rcode = sp.call(cmd, stdout = fo)
ftime = monotonic_time()
sys.stdout.write("query finished\n")
elapsed = ftime - stime
with open(timefile, "w") as ft:
ft.write("{0}\n".format(elapsed))
if not rcode: break
else:
count += 1
sys.stderr.write("Query Execution Error : {0}\n".format(rcode))
time.sleep(5)
finally:
fo.close()
if self.profflg:
pio.kill()
pcpu.kill()
if self.iotraceflg: pbt.kill()
return elapsed
def removeExpiredPeers(self):
conn = self._getRedisConn()
for info_hash in conn.sunion(Peers.SEEN_TORRENTS):
self.log.info('Checking for expiry of peers @ %s',info_hash.encode('hex'))
for peerNumber,lastSeenTime in conn.hgetall(info_hash + Peers.LAST_SEEN_SUFFIX).iteritems():
currentTime = monotonic_time()
if currentTime - float(lastSeenTime) >= self.peerExpirationPeriod:
conn.hdel(info_hash,peerNumber)
#if removing the peer causes the hash set to no longer
#exist, then remove it from the set of observed torrents
if not conn.exists(info_hash):
conn.srem(Peers.SEEN_TORRENTS, info_hash)
self.log.info('Expired peer')
def removeExpiredPeers(self):
conn = self._getRedisConn()
for info_hash in conn.sunion(Peers.SEEN_TORRENTS):
self.log.info('Checking for expiry of peers @ %s',
info_hash.encode('hex'))
for peerNumber, lastSeenTime in conn.hgetall(
info_hash + Peers.LAST_SEEN_SUFFIX).iteritems():
currentTime = monotonic_time()
if currentTime - float(
lastSeenTime) >= self.peerExpirationPeriod:
conn.hdel(info_hash, peerNumber)
#if removing the peer causes the hash set to no longer
#exist, then remove it from the set of observed torrents
if not conn.exists(info_hash):
conn.srem(Peers.SEEN_TORRENTS, info_hash)
self.log.info('Expired peer')
def evict(self):
envs = self.kube.get_svc_list()
env_ids = filter(lambda n: n.startswith(self.kube.ENV_PREFIX),
[e.metadata.name for e in envs.items])
now = monotonic_time()
logging.warning("eviction start:%s", env_ids)
# terminate outdated envs
for env_id in env_ids:
if env_id not in self.keep_alives:
self.keep_alives[env_id] = now # ensure every env_id appears in keep_alives at least once
elif now - self.keep_alives[env_id] > self.TIMEOUT:
# deadline passed
logging.warning("timeout for %s: %s", env_id, (now - self.keep_alives[env_id]))
self.kube.destroy_env(env_id)
# remove desolated keepalives
for env_id in self.keep_alives.keys():
if env_id not in env_ids:
self.keep_alives.pop(env_id)
def updatePeer(self,info_hash,peer): peerNumber = self.getPeerNumber(peer) conn = self._getRedisConn() wasSeed = conn.hget(info_hash, peerNumber) if wasSeed == None: wasSeed = False else: wasSeed = wasSeed == '1' isSeed = peer.left == 0 isAdd = 1 == conn.hset(info_hash, peerNumber,'1' if isSeed else '0') conn.hset(info_hash + Peers.LAST_SEEN_SUFFIX,peerNumber,monotonic_time()) conn.sadd(Peers.SEEN_TORRENTS,info_hash) #If the peer was added or #If the peer changed from seed to leech, or from leech #to seed then the count has changed change = isAdd or (wasSeed!=isSeed) return change
def updatePeer(self, info_hash, peer):
peerNumber = self.getPeerNumber(peer)
conn = self._getRedisConn()
wasSeed = conn.hget(info_hash, peerNumber)
if wasSeed == None:
wasSeed = False
else:
wasSeed = wasSeed == '1'
isSeed = peer.left == 0
isAdd = 1 == conn.hset(info_hash, peerNumber, '1' if isSeed else '0')
conn.hset(info_hash + Peers.LAST_SEEN_SUFFIX, peerNumber,
monotonic_time())
conn.sadd(Peers.SEEN_TORRENTS, info_hash)
#If the peer was added or
#If the peer changed from seed to leech, or from leech
#to seed then the count has changed
change = isAdd or (wasSeed != isSeed)
return change
def keepalive(self, env_id):
self.keep_alives[env_id] = monotonic_time()
def run_query_wperf(self, destdir):
# define output file names
exppath = "{0}/{1}.exp".format(destdir, self.fprefix)
resfile = "{0}/{1}.res".format(destdir, self.fprefix)
timefile = "{0}/{1}.time".format(destdir, self.fprefix)
perfpath = "{0}/{1}.perf".format(destdir, self.fprefix)
if self.profflg:
cpupath = "{0}/{1}.cpu".format(destdir, self.fprefix)
iopath = "{0}/{1}.io".format(destdir, self.fprefix)
count = 0
finishflg = False
while not finishflg and count < 3:
self.restart_db(destdir)
# create db connection
conn = pg.connect(database = self.db, user = self.user)
cur = conn.cursor()
# output execution plan
cur.execute("explain " + self.query)
with open(exppath, "w") as fo:
fo.writelines([vals[0] + "\n" for vals in cur.fetchall()])
# get backend pid
cur.execute("select pg_backend_pid()")
pgpid = cur.fetchone()[0]
sys.stdout.write("query started: trycount {0}\n".format(count))
if self.iotraceflg:
pbt = sp.Popen(["blktrace", "/dev/md0", "-D", destdir])
if self.profflg:
# run mpstat and iostat
pcpu = sp.Popen(["mpstat", "-P", "ALL", "1"], stdout = open(cpupath, "w"))
pio = sp.Popen(["iostat", "-x", "1"], stdout = open(iopath, "w"))
try:
# run query
pid = os.fork()
if 0 == pid:
# perf process
cmd = shlex.split(self.perfcmd.format(pid = pgpid, output = perfpath))
p = sp.Popen(cmd)
def stopperf(signum, frame):
os.kill(p.pid, signal.SIGINT)
ret = p.wait()
print ret, open(perfpath).readlines()
os.kill(os.getpid(), signal.SIGKILL)
signal.signal(signal.SIGINT, stopperf)
p.wait()
else:
stime = monotonic_time()
cur.execute(self.query)
ftime = monotonic_time()
os.kill(pid, signal.SIGINT)
sys.stdout.write("query finished\n")
finishflg = True
except pg.Error, e:
errstate = errorcodes.lookup(e.pgcode)
sys.stderr.write("query execution failed: {0}\n".format(errstate))
count += 1
time.sleep(5)
continue
except Exception, e:
sys.stderr.write("query execution failed: {0}\n".format(e.message))
count += 1
time.sleep(5)
continue
def __enter__(self):
self.start = monotonic_time()
return self
def __exit__(self, *args):
self.end = monotonic_time()
self.interval = self.end - self.start