def __init__(self, api=MongoStampedAPI(), logsQuery=logsQuery()):
self.stamp_collection = api._stampDB._collection
self.acct_collection = api._userDB._collection
self.query = logsQuery
self.writer = statWriter("dashboard")
conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
self.domain = conn.get_domain("dashboard")
class SimpleDBBackend:
def __init__(self):
self.connection = SDBConnection(aws_credentials.accessKey,
aws_credentials.secretKey)
self.domain = self.connection.get_domain(TABLE_NAME)
def put(self, key, value):
sampler.begin()
try:
self.domain.put_attributes(key, {VALUE:value})
finally:
sampler.end()
def get(self, key):
sampler.begin()
try:
# First try an eventually consistent read.
result = self.domain.get_attributes(key, consistent_read=False)
return result[VALUE]
except KeyError:
# The eventually consistent read failed. Try a strongly consistent
# read.
result = self.domain.get_attributes(key, consistent_read=True)
return result[VALUE]
finally:
sampler.end()
def incRefCount(self, key):
# Not implemented.
pass
def decRefCount(self, key):
# Not implemented.
pass
def nuke(self):
# Delete and re-create the table.
self.connection.delete_domain(TABLE_NAME)
self.domain = self.connection.create_domain(TABLE_NAME)
def flush(self):
pass # No-op.
def submit():
if len(sys.argv) < 3:
print "Usage:"
print " %s -seed <seed> [<spawn coords>]" % sys.argv[0]
return
# TODO use less crappy command line parsing
seed = sys.argv[2]
if len(sys.argv) == 4:
spawn = [int(x) for x in sys.argv[3].split(",")]
assert(len(spawn) == 3)
print "Generate world with this seed (\"%s\") with spawn %r [y/N]?" % (seed, spawn)
else:
spawn = None
print "Generate world with this seed (\"%s\") [y/N]?" % seed
if raw_input().lower() == 'y':
uid = uuid.uuid4()
print "Submitting job %s to queue..." % uid
sqs = SQSConnection()
sdb = SDBConnection()
queue = sqs.get_queue("overviewer-genfromseed")
db = sdb.get_domain("overviewerdb")
print queue
print db
data = dict()
data['uuid'] = str(uid)
data['seed'] = seed
data['generated'] = False
if spawn:
data['target_spawn'] = spawn
if not db.put_attributes(uid, data):
print "***Error: Failed to update the db"
return 1
msg = Message()
msg.set_body(str(uid))
if not queue.write(msg):
print "***Error: Failed to enqueue"
return 1
print "Ok, job enqueued"
else:
print "Ok, not submitting. Bye"
return
def submit():
if len(sys.argv) < 3:
print "Usage:"
print " %s -submit <world uuid>" % sys.argv[0]
return
sdb = SDBConnection()
db = sdb.get_domain("overviewerdb")
# TODO use less crappy command line parsing
world_uuid = uuid.UUID(sys.argv[2])
world_item = db.get_item(world_uuid)
if not world_item:
print "Can't find that world!"
return 1
print "Submit this world for rendering? [y/N]"
if raw_input().lower() != 'y':
return "Ok, nevermind."
return 0
from boto.sqs.connection import SQSConnection
sqs = SQSConnection()
queue = sqs.get_queue("overviewer-render")
render_uuid = uuid.uuid4()
print "Render UUID:", render_uuid
data = dict()
data['uuid'] = str(render_uuid)
data['rendered'] = False
data['world_uuid'] = str(world_uuid)
if not db.put_attributes(str(render_uuid), data):
print "***Error: Failed to update the db"
return 1
msg = Message()
msg.set_body(str(render_uuid))
if not queue.write(msg):
print "***Error: Failed to enqueue"
return 1
print "Ok, job enqueued"
return 0
def add_from_url():
if len(sys.argv) < 3:
print "Usage:"
print " %s -url <url>" % sys.argv[0]
return
url = sys.argv[2]
print "Generate world with this url (\"%s\") [y/N]?" % url
if raw_input().lower() != 'y':
print "Ok, nevermind"
return
uid = uuid.uuid4()
data = dict()
data['uuid'] = uid
data['world_url'] = url
sdb = SDBConnection()
db = sdb.get_domain("overviewerdb")
if not db.put_attributes(uid, data):
print "***Error: Failed to update the db"
return 1
print "Ok. DB updated"
print uid
class statWriter(object):
def __init__(self,domain=None):
self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
self.domain_name = domain
def write(self,stat,key=None,domain=None):
if domain is None:
if self.domain_name is None:
return False
else:
domain = self.domain_name
if key is None:
key = uuid.uuid1()
if len(stat) > 0:
try:
domain = self.conn.get_domain(domain)
except SDBResponseError:
domain = self.conn.create_domain(domain)
domain.put_attributes(key, stat, replace=False)
return True
#!/usr/bin/python2
import sys
import os
from pprint import pprint
try:
import boto
except ImportError:
sys.path.append(os.path.expanduser("~/devel/boto"))
import boto
from boto.sdb.connection import SDBConnection
uid = sys.argv[1]
print "Looking up", uid
sdb = SDBConnection()
db = sdb.get_domain("overviewerdb")
data = db.get_item(uid)
pprint(data)
class SimpleDB(object):
def __init__(self, domain=None):
self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
self.domain_name = None
self.domains = {}
if domain is None and is_ec2():
stack = get_stack()
stack_name = str(stack['instance']['stack'])
self.domain_name = stack_name
elif domain is not None:
self.domain_name = domain
def addStat(self, stat):
if self.domain_name is None:
return
try:
# Only add specific parameters
data = {}
if 'user_id' in stat:
data['uid'] = stat['user_id']
if 'path' in stat:
data['uri'] = stat['path']
if 'method' in stat:
data['mtd'] = stat['method']
if 'form' in stat:
try:
for k, v in stat['form'].items():
try:
if not isinstance(v, basestring):
v = str(v)
if len(v.encode('utf-8')) > 1024:
v = '<INPUT TOO LONG>'
data['frm_%s' % k] = v
except Exception as e:
print e
except Exception as e:
print e
if 'result' in stat:
data['cde'] = str(stat['result'])
if 'begin' in stat:
data['bgn'] = stat['begin'].isoformat()
if 'finish' in stat:
data['end'] = stat['finish'].isoformat()
if 'node' in stat:
data['nde'] = stat['node']
if 'client_id' in stat:
data['cid'] = stat['client_id']
if 'duration' in stat:
data['dur'] = "{0:10d}".format(stat['duration'])
if len(data) > 0:
statId = str(ObjectId())
if data['uri'] != '/v1/ping.json' and data['uri'] != '/v1/temp/ping.json':
suffix = '0%s' % (sha1(statId).hexdigest()[0])
if suffix in self.domains:
domain = self.domains[suffix]
else:
try:
domain = self.conn.get_domain('%s_%s' % (self.domain_name, suffix))
except SDBResponseError:
domain = self.conn.create_domain('%s_%s' % (self.domain_name, suffix))
self.domains[suffix] = domain
domain.put_attributes(statId, data, replace=False)
except Exception as e:
print e
raise
def add_from_path():
if len(sys.argv) < 3:
print "Usage:"
print " %s -path <path>" % sys.argv[0]
return
path = sys.argv[2]
if os.path.isdir(path):
print "You've specified a directory. I'll tar it up before uploading"
print "OK? [y/N] ",
if raw_input().lower() != 'y':
print "Ok, nevermind."
return
tmpdir = tempfile.mkdtemp(prefix="mc_gen")
print "tmpdir is", tmpdir
print "Making tarball..."
p = subprocess.Popen(["tar", "-cf", os.path.join(tmpdir, "world.tar"), "."],
cwd=path)
p.wait()
if p.returncode != 0:
print "***Error: tar failed"
return
print "OK."
print "Compressing..."
p = subprocess.Popen(["bzip2", "world.tar"],
shell=False,
cwd=tmpdir)
p.wait()
if p.returncode != 0:
print "***Error: compress failed"
return
print "OK."
print "Checking filesize..."
s = os.stat(os.path.join(tmpdir, "world.tar.bz2"))
if s.st_size > 10*1024*1024:
print "***Error: Compressed world is too big"
return 1
print "OK."
uid = uuid.uuid4()
print uid
s3 = S3Connection()
bucket = s3.get_bucket("overviewer-worlds")
k = Key(bucket)
k.key = "%s.tar.bz2" % uid
print "Uploading to S3..."
k.set_contents_from_filename(os.path.join(tmpdir, "world.tar.bz2"), reduced_redundancy=True)
print "OK."
k.make_public()
urlbase = "https://s3.amazonaws.com/overviewer-worlds/"
url = urlbase + k.key
print "World is now available at:", url
data = dict()
data['uuid'] = uid
data['world_url'] = url
sdb = SDBConnection()
db = sdb.get_domain("overviewerdb")
if not db.put_attributes(uid, data):
print "***Error: Failed to update the db"
return 1
print "Ok. DB updated"
elif os.path.isfile(path):
print "You've specified a file. I'll upload it without modification"
print "OK? [y/N] ",
if raw_input().lower() != 'y':
print "Ok, nevermind."
return
uid = uuid.uuid4()
print uid
s3 = S3Connection()
bucket = s3.get_bucket("overviewer-worlds")
k = Key(bucket)
k.key = "%s.tar.bz2" % uid
print "Uploading to S3..."
k.set_contents_from_filename(path, reduced_redundancy=True)
print "OK."
k.make_public()
urlbase = "https://s3.amazonaws.com/overviewer-worlds/"
url = urlbase + k.key
print "World is now available at:", url
data = dict()
data['uuid'] = uid
data['world_url'] = url
sdb = SDBConnection()
db = sdb.get_domain("overviewerdb")
if not db.put_attributes(uid, data):
print "***Error: Failed to update the db"
return 1
print "Ok. DB updated"
else:
print "Sorry, I can't find that."
return 1
def generate():
pass
from boto.sqs.connection import SQSConnection
sqs = SQSConnection()
queue = sqs.get_queue("overviewer-genfromseed")
sdb = SDBConnection()
db = sdb.get_domain("overviewerdb")
message = queue.read(visibility_timeout=15)
if not message:
print "Nothing in the queue. Please try again later"
return 0
uid = message.get_body()
print "Got a job for %r" % uid
data = db.get_item(uid)
if 'target_spawn' in data:
data['target_spawn'] = map(int, data['target_spawn'])
print data
# this script generate maps from seeds
# if this map is already generated, then upate the db
if data['generated'] == 'True':
print "---Warning: I was asked to generate this map, but it's already generated"
print " I'm going to update the db, but not re-generate"
data['generated'] = True
data.save()
queue.delete_message(message)
return
# check config options
if not os.path.isfile(config.minecraft_server):
raise Exception("minecraft_server isn't configured")
# with a generous amount of rounding up, assume it'll take 5 minutes to generate the map
message.change_visibility(5*60)
tmpdir = tempfile.mkdtemp(prefix="mc_gen")
# create a settings.properties file with our seed
with open(os.path.join(tmpdir, "server.properties"), "w") as f:
f.write("level-seed=%s" % data['seed'])
p = subprocess.Popen(["java", "-jar",
config.minecraft_server, "-noGUI"],
shell=False,
stdin=subprocess.PIPE,
cwd=tmpdir)
p.stdin.write("stop\n")
p.stdin.close()
p.wait()
print ""
print "Minecraft server exited with %r" % p.returncode
print "World resided in %r" % tmpdir
# if we want a specific spawn, we need to rewrite the level.dat file,
# remove the old region files, and restart the server
if 'target_spawn' in data:
s = data['target_spawn']
leveldat = redstone.NBT.parse_from_file(os.path.join(tmpdir, "world", "level.dat"))
root = leveldat.root
root['Data']['SpawnX'].set_integer(int(s[0]))
root['Data']['SpawnY'].set_integer(int(s[1]))
root['Data']['SpawnZ'].set_integer(int(s[2]))
leveldat.write_to_file(os.path.join(tmpdir, "world", "level.dat"))
shutil.rmtree(os.path.join(tmpdir,"world","region"))
p = subprocess.Popen(["java", "-jar",
config.minecraft_server, "-noGUI"],
shell=False,
stdin=subprocess.PIPE,
cwd=tmpdir)
p.stdin.write("stop\n")
p.stdin.close()
p.wait()
print "Minecraft server exited with %r" % p.returncode
message.change_visibility(5*60)
print "Making tarball..."
p = subprocess.Popen(["tar", "-cf", "world.tar", "world/"],
cwd=tmpdir)
p.wait()
if p.returncode != 0:
print "***Error: tar failed"
return
print "OK."
print "Compressing..."
p = subprocess.Popen(["bzip2", "world.tar"],
shell=False,
cwd=tmpdir)
p.wait()
if p.returncode != 0:
print "***Error: compress failed"
return
print "OK."
s3 = S3Connection()
bucket = s3.get_bucket("overviewer-worlds")
k = Key(bucket)
k.key = "%s.tar.bz2" % uid
print "Uploading to S3..."
k.set_contents_from_filename(os.path.join(tmpdir, "world.tar.bz2"), reduced_redundancy=True)
print "OK."
k.make_public()
urlbase = "https://s3.amazonaws.com/overviewer-worlds/"
url = urlbase + k.key
print "World is now available at:", url
data['generated'] = True
data['world_url'] = url
data.save()
print "Database updated."
queue.delete_message(message)
print "All done!"
class weeklyScore(object):
def __init__(self,api):
self.api = api
self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
self.domains = {}
self.statDict = {}
for i in range (0,16):
suffix = '0'+hex(i)[2]
self.domains[suffix] = self.conn.get_domain('stats_dev_%s' % (suffix))
def scoreQuery(self,domain,t0,t1,uri,points):
query = 'select uri,uid from `%s` where uri = "%s" and bgn >= "%s" and bgn <= "%s"' % (domain.name,uri,t0.isoformat(),t1.isoformat())
stats = domain.select(query)
for stat in stats:
if 'uid' not in stat or 'uri' not in stat or 'cde' in stat:
continue
key = stat['uid']
try:
self.statDict[key] += points
except KeyError:
self.statDict[key] = points
def segmentationReport(self,t0,t1,bMonth,percentage=False):
self.statDict = {}
scores = {
#2 points for likes
'/v1/stamps/likes/create.json': 2,
'/v1/stamps/likes/remove.json': -2,
#2 points for todos
'/v1/todos/create.json': 2,
'/v1/todos/remove.json': -2,
#3 points for comments
'/v1/comments/create.json': 3,
'/v1/comments/remove.json': -3,
#3 points for adding a friend
'/v1/friendships/create.json': 3,
'/v1/friendships/remove.json': -3,
#3 points for taking an action
'/v1/actions/complete.json': 2,
#5 points for a stamp
'/v1/stamps/create.json': 5,
'/v1/stamps/remove.json': -5,
#5 points for an invite
'/v1/frienships/invite.json': 5,
}
pool = Pool(30)
for i in range (0,16):
suffix = '0'+hex(i)[2]
for uri in scores:
pool.spawn(self.scoreQuery,self.domains[suffix],t0,t1,uri,scores[uri])
pool.join()
power = 0
active = 0
irregular = 0
dormant = 0
agg_score = 0
if bMonth:
POWER = 100
AVERAGE = 10
else: #Looking at a week
POWER = 25
AVERAGE = 5
for uid in self.statDict:
agg_score += self.statDict[uid]
if self.statDict[uid] >= POWER:
power += 1
elif self.statDict[uid] >= AVERAGE:
active += 1
elif self.statDict[uid] > 0:
irregular += 1
else:
dormant +=1
users = len(self.statDict)
if users == 0:
return 0,0,0,0,0,0
if percentage:
power = float(power)/users*100
active = float(active) / users*100
irregular = float(irregular)/users*100
dormant = float(dormant)/users*100
mean_score = float(agg_score) / users
return users,power,active,irregular,dormant,mean_score
class SimpleDBConnection(object):
def __init__(self, stack):
try:
self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
except SDBResponseError:
print "SimpleDB Connection Refused"
raise
self._stack = stack
@lazyProperty
def domains(self):
result = []
for i in range(16):
suffix = "0" + hex(i)[2]
name = "%s_%s" % (self._stack, suffix)
result.append(self.conn.get_domain(name))
return result
def _queryParams(self, params, bgn, end):
queryParams = []
if params is not None:
for k, v in params.items():
queryParams.append("%s='%s'" % (k, v))
if bgn is not None:
queryParams.append("end >= '%s'" % bgn.isoformat())
if end is not None:
queryParams.append("end < '%s'" % end.isoformat())
return queryParams
def _deserialize(self, item):
def isoparse(s):
# Credit: http://blog.client9.com/2010/02/fast-iso-datetime-format-parsing-in.html
try:
return datetime(int(s[0:4]), int(s[5:7]), int(s[8:10]), int(s[11:13]), int(s[14:16]), int(s[17:19]))
except:
return None
# Convert datetimes
if "bgn" in item:
item["bgn"] = isoparse(item["bgn"])
if "end" in item:
item["end"] = isoparse(item["end"])
# Convert numbers
if "dur" in item:
item["dur"] = int(item["dur"])
return item
def query(self, params=None, fields=None, bgn=None, end=None):
queryFields = "*"
if fields is not None:
queryFields = ",".join(fields)
queryParams = self._queryParams(params, bgn, end)
pool = Pool(len(self.domains))
data = []
def run(domain, query):
data.extend(domain.select(query))
for domain in self.domains:
# Build query
query = "select %s from `%s`" % (queryFields, domain.name)
if len(queryParams) > 0:
query += " where %s" % (" and ".join(queryParams))
# Run it
pool.spawn(run, domain, query)
pool.join()
results = []
for item in data:
results.append(self._deserialize(item))
return results
def count(self, params=None, bgn=None, end=None):
queryParams = self._queryParams(params, bgn, end)
pool = Pool(len(self.domains))
data = []
def run(domain, query):
data.extend(domain.select(query))
for domain in self.domains:
# Build query
query = "select count(*) from `%s`" % domain.name
if len(queryParams) > 0:
query += " where %s" % (" and ".join(queryParams))
# Run it
pool.spawn(run, domain, query)
pool.join()
count = 0
for item in data:
count += int(item.pop("Count", 0))
return count
def render():
# check config options
if not os.path.isdir(config.overviewer_root):
raise Exception("overviewer_root isn't configured")
if not os.path.isfile(config.upload_ssh_key):
raise Exception("upload_ssh_key isn't configured")
sqs = SQSConnection()
queue = sqs.get_queue("overviewer-render")
sdb = SDBConnection()
db = sdb.get_domain("overviewerdb")
message = queue.read(visibility_timeout=15)
if not message:
print "Nothing in the queue. Please try again later"
return 0
render_uuid = message.get_body()
print "render uuid:", render_uuid
render_item = db.get_item(str(render_uuid))
if not render_item:
print "***Error can't find a world with that UUID"
return 1
if render_item.get("rendered") != "False":
print "***Error: this render has already been started"
print " state", render_item.get("rendered")
return 1
world_uuid = render_item.get("world_uuid")
world_item = db.get_item(str(world_uuid))
print "world uuid:", world_uuid
url = world_item.get("world_url", None)
if not url:
print "***Error: can't find worldurl"
return 1
message.change_visibility(3*60)
render_item['rendered'] = "inprogress"
render_item.save()
print "Getting map..."
map_url = urllib2.urlopen(url)
print "OK."
tmpdir = tempfile.mkdtemp(prefix="mc_gen")
fobj = open(os.path.join(tmpdir, "world.tar.bz2"), "w")
print "Downloading map to %s..." % tmpdir
shutil.copyfileobj(map_url, fobj)
fobj.close()
print "OK."
print "Uncompressing..."
os.mkdir(os.path.join(tmpdir, "world"))
p = subprocess.Popen(["tar", "-jxf", os.path.join(tmpdir,"world.tar.bz2")],
cwd=os.path.join(tmpdir,"world"))
p.wait()
if p.returncode != 0:
print "***Error: decompressing"
return 1
# find the exact directory containing level.dat
def findLevel(start):
for root, dirs, files in os.walk(start):
if "level.dat" in files: return root
for d in dirs:
findLevel(os.path.join(root, d))
raise Exception("Failed to find level.dat")
real_world_dir = findLevel(os.path.join(tmpdir, "world"))
# TODO message.change_visibility(10*60)
p = subprocess.Popen(["python2",
os.path.join(config.overviewer_root, "overviewer.py"),
real_world_dir,
os.path.join(tmpdir, "output_dir"),
"--rendermode=smooth-lighting"])
p.wait()
if p.returncode != 0:
print "***Error: rendering"
return 1
print "Making tarball..."
p = subprocess.Popen(["tar", "-cf",
os.path.join(tmpdir, "render.tar"),
"."],
cwd=os.path.join(tmpdir, "output_dir"))
p.wait()
if p.returncode != 0:
print "***Error: tar failed"
return
print "OK."
print "Compressing..."
p = subprocess.Popen(["bzip2", "render.tar"],
shell=False,
cwd=tmpdir)
p.wait()
if p.returncode != 0:
print "***Error: compress failed"
return
print "OK."
message.change_visibility(5*60)
print "Uploading to overviewer.org..."
p = subprocess.Popen(["ssh",
"-l", "upload",
"-i", config.upload_ssh_key,
"new.overviewer.org",
render_uuid],
stdin=subprocess.PIPE)
fobj = open(os.path.join(tmpdir, "render.tar.bz2"))
shutil.copyfileobj(fobj, p.stdin)
p.stdin.close()
p.wait()
if p.returncode != 0:
print "***Error: uploading"
return 1
print "OK"
render_item['rendered'] = "True"
render_item.save()
print "Database updated"
queue.delete_message(message)
class SimpleDBConnection(object):
def __init__(self):
try:
self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
except SDBResponseError:
print "SimpleDB Connection Refused"
self.statList = []
self.statCount = 0
self.v1_prod_domains = {}
self.v2_prod_domains = {}
self.v2_dev_domains = {}
self.v2_stress_domains = {}
for i in range (0,16):
suffix = '0'+hex(i)[2]
self.v1_prod_domains[suffix] = self.conn.get_domain('stats_prod_%s' % suffix)
self.v2_prod_domains[suffix] = self.conn.get_domain('bowser_%s' % suffix)
self.v2_dev_domains[suffix] = self.conn.get_domain('stats_dev_%s' % suffix)
self.v2_stress_domains[suffix] = self.conn.get_domain('stress_%s' % suffix)
# Perform a query and write to an optional list, set or dict (key is first field specified)
def execute(self, domain, query_dict, fields, bgn=None, end=None, destination=None, limit=None, duration=None):
query = 'select %s from `%s`' % (fields,domain.name)
transition = 'where'
for key,value in query_dict.items():
if "!=" in value:
query = '%s %s %s != "%s"' % (query, transition, key, value[2:])
else:
query = '%s %s %s="%s"' % (query, transition, key, value)
transition = 'and'
if bgn is not None:
query = '%s %s bgn > "%s"' % (query, transition, bgn.isoformat())
transition = 'and'
if end is not None:
query = '%s %s bgn > "%s"' % (query, transition, bgn.isoformat())
transition = 'and'
init_results = domain.select(query)
results = reveal(init_results,limit=limit,duration=duration)
if destination != None:
if fields == 'count(*)':
destination += results[0]
else:
destination.extend(results)
return results
def query (self, stack, query_dict, fields='*', bgn=None, end=None, limit=None, duration=None):
if fields == 'count(*)':
self.statCount = 0
destination = self.statCount
else:
self.statList = []
destination = self.statList
if stack == 'bowser':
domains = self.v2_prod_domains
else:
domains = self.v2_dev_domains
pool = Pool(16)
for i in range (0,16):
suffix = '0'+hex(i)[2]
pool.spawn(self.execute, domains[suffix], query_dict, fields=fields, bgn=bgn,end=end, destination=destination, limit=limit/16 + 1, duration=duration)
pool.join()
if fields == 'count(*)':
return self.statCount
else:
return self.statList[:limit]
class logsQuery(object):
def __init__(self,domain_name=None):
self.conn = SDBConnection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY)
self.domains = {}
self.statSet = set()
self.statDict = {}
self.errDict = {}
self.statCount = 0
self.statCountByNode = {}
self.statTimeByNode = {}
if domain_name is None:
domain_name = 'stress'
for i in range (0,16):
suffix = '0'+hex(i)[2]
self.domains[suffix] = self.conn.get_domain('%s_%s' % (domain_name,suffix))
def performQuery(self,domain,fields,uri,t0,t1,byNode=False):
if uri is not None:
query = 'select %s from `%s` where uri = "%s" and bgn >= "%s" and bgn <= "%s"' % (fields, domain.name, uri, t0.isoformat(), t1.isoformat())
else:
query = 'select %s from `%s` where bgn >= "%s" and bgn <= "%s"' % (fields, domain.name, t0.isoformat(), t1.isoformat())
stats = domain.select(query)
for stat in stats:
try:
if fields == 'count(*)':
self.statCount += int(stat['Count'])
elif byNode:
bgn = stat['bgn'].split('T')
end = stat['end'].split('T')
if end[0] == bgn[0]:
bgn = bgn[1].split(':')
end = end[1].split(':')
hours = float(end[0]) - float(bgn[0])
minutes = float(end[1]) - float(bgn[1])
seconds = float(end[2]) - float(bgn[2])
diff = seconds + 60*(minutes + 60*hours)
try:
self.statCountByNode[stat['nde']] += 1
self.statTimeByNode[stat['nde']] += diff
except KeyError:
self.statCountByNode[stat['nde']] = 1
self.statTimeByNode[stat['nde']] = diff
else:
self.statSet.add(stat[fields])
except KeyError:
pass
def activeUsers(self,t0, t1):
self.statSet = set()
pool = Pool(16)
for i in range (0,16):
suffix = '0'+hex(i)[2]
#Just use collections inbox for speed
pool.spawn(self.performQuery,self.domains[suffix],'uid',"/v1/activity/unread.json",t0,t1)
pool.join()
return len(self.statSet)
def latencyQuery(self,domain,t0,t1,uri,blacklist,whitelist):
if uri is None:
query = 'select uri,frm_scope,bgn,end,cde,uid from `%s` where uri like "/v1/%%" and bgn >= "%s" and bgn <= "%s"' % (domain.name,t0.isoformat(),t1.isoformat())
else:
query = 'select uri,frm_scope,bgn,end,cde,uid from `%s` where uri = "%s" and bgn >= "%s" and bgn <= "%s"' % (domain.name,uri,t0.isoformat(),t1.isoformat())
stats = domain.select(query)
for stat in stats:
if 'uid' in stat and stat['uid'] in blacklist:
continue
elif len(blacklist) == 0 and len(whitelist) > 0 and 'uid' in stat and stat['uid'] not in whitelist:
continue
bgn = stat['bgn'].split('T')
end = stat['end'].split('T')
if end[0] == bgn[0]:
bgn = bgn[1].split(':')
end = end[1].split(':')
hours = float(end[0]) - float(bgn[0])
minutes = float(end[1]) - float(bgn[1])
seconds = float(end[2]) - float(bgn[2])
diff = seconds + 60*(minutes + 60*hours)
key = stat['uri']
if 'frm_scope' in stat:
key = "%s?scope=%s" % (stat['uri'], stat['frm_scope'])
if 'cde' in stat:
errType = stat['cde'][0]
try:
self.errDict['%s-%s' % (key,errType)] +=1
except KeyError:
self.errDict['%s-%s' % (key,errType)] = 1
else:
try:
self.statDict[key].append(diff)
except KeyError:
self.statDict[key] = [diff]
def latencyReport(self,t0,t1,uri=None,blacklist=[],whitelist=[]):
self.statDict = {}
self.errDict = {}
pool = Pool(16)
for i in range (0,16):
suffix = '0'+hex(i)[2]
pool.spawn(self.latencyQuery,self.domains[suffix],t0,t1,uri,blacklist,whitelist)
pool.join()
for uri in self.statDict:
sum = 0
max = 0
for num in self.statDict[uri]:
sum += num
if num > max:
max = num
mean = float(sum) / len(self.statDict[uri])
sorte = sorted(self.statDict[uri])
median = percentile(sorte,.5)
ninetieth = percentile(sorte,.9)
n = len(self.statDict[uri])
errors4 = 0
errors5 = 0
if uri+'-4' in self.errDict:
errors4 = self.errDict[uri+'-4']
if uri+'-5' in self.errDict:
errors5 = self.errDict[uri+'-5']
self.statDict[uri] = '%.3f' % mean,'%.3f' % median,'%.3f' % ninetieth, '%.3f' % max, n, errors4,errors5
return self.statDict
def dailyLatencyReport(self,t0,t1,uri,blacklist,whitelist):
self.statDict = {}
diff = (t1 - t0).days + 1
output = []
for i in range (0,diff):
t2 = today(t0+datetime.timedelta(days=i+1))
t3 = today(t0+datetime.timedelta(days=i+2))
self.statDict = {}
self.errDict = {}
pool = Pool(16)
for k in range (0,16):
suffix = '0'+hex(k)[2]
pool.spawn(self.latencyQuery,self.domains[suffix],t2,t3,uri,blacklist,whitelist)
pool.join()
for uri in self.statDict:
sum = 0
max = 0
for num in self.statDict[uri]:
sum += num
if num > max:
max = num
mean = float(sum) / len(self.statDict[uri])
sorte = sorted(self.statDict[uri])
median = percentile(sorte,.5)
ninetieth = percentile(sorte,.9)
n = len(self.statDict[uri])
errors4 = 0
errors5 = 0
if uri+'-4' in self.errDict:
errors4 = self.errDict[uri+'-4']
if uri+'-5' in self.errDict:
errors5 = self.errDict[uri+'-5']
output.append((t2.date().isoformat(),'%.3f' % mean,'%.3f' % median,'%.3f' % ninetieth, '%.3f' % max, n, errors4,errors5))
return output
def qpsReport(self,time,interval,total_seconds):
blacklist=[]
whitelist=[]
count_report = {}
mean_report = {}
t0 = time - datetime.timedelta(0,total_seconds)
for i in range (0,total_seconds/interval):
self.statCountByNode = {}
self.statTimeByNode = {}
t1 = t0 + datetime.timedelta(0,i*interval)
t2 = t0 + datetime.timedelta(0,(i+1)*interval)
pool = Pool(32)
for j in range (0,16):
suffix = '0'+hex(j)[2]
pool.spawn(self.performQuery,self.domains[suffix],'nde,bgn,end',None,t1,t2,byNode=True)
pool.join()
for node in self.statCountByNode:
count = self.statCountByNode[node]
mean = float(self.statTimeByNode[node])/count
try:
while len(count_report[node]) < i:
count_report[node].insert(0,0)
mean_report[node].insert(0,0)
count_report[node].insert(0,"%.3f" % (float(count)/interval))
mean_report[node].insert(0,"%.3f" % (mean))
except KeyError:
count_report[node] = [0]*i
mean_report[node] = [0]*i
count_report[node].insert(0,"%.3f" % (float(count)/interval))
mean_report[node].insert(0,"%.3f" % (mean))
for node in count_report:
while len(count_report[node]) < total_seconds/interval:
count_report[node].insert(0,0)
mean_report[node].insert(0,0)
return count_report,mean_report
def customQuery(self,t0,t1,fields,uri):
if fields == 'count(*)':
self.statCount = 0
else:
self.statSet = set()
pool = Pool(16)
for i in range (0,16):
suffix = '0'+hex(i)[2]
pool.spawn(self.performQuery,self.domains[suffix],fields,uri,t0,t1)
pool.join()
if fields == 'count(*)':
return self.statCount
else:
return len(self.statSet)
class Catalog:
DOMAIN = "cloudvision"
CATALOG_ITEM = "catalog"
NEXT_TABLE_ID_ATTR = "next_table_id"
TABLES_ATTR = "tables"
TABLES_DELIM = ","
TABLES_ASSIGN = ":"
TABLE_NAME_ATTR = "name"
def __init__(self):
self.conn = None
self.dom = None
def connect(self):
self.conn = SDBConnection()
self.dom = self.conn.get_domain(self.DOMAIN)
def get_tables(self):
tables = [ ]
tables_str = self.get_tables_assign()
for t in tables_str:
tname, tid = t.split(self.TABLES_ASSIGN)
tid = int(tid)
tables.append(Table(tid, tname))
return tables
def get_tables_assign(self):
catalog = self.dom.get_item(self.CATALOG_ITEM)
if len(catalog[self.TABLES_ATTR]) > 0:
return catalog[self.TABLES_ATTR].split(self.TABLES_DELIM)
return [ ]
def get_table_by_name(self, name):
assert name is not None
tables = self.get_tables()
for t in tables:
if t.get_name() == name:
return t
return None
def get_table_by_id(self, id):
assert id is not None
tables = self.get_tables()
for t in tables:
if t.get_id() == id:
return t
return None
def put_table(self, name):
assert name is not None
catalog = self.dom.get_item(self.CATALOG_ITEM)
tables = self.get_tables_assign()
for t in tables:
tname, tid = t.split(self.TABLES_ASSIGN)
assert tname != name
id = int(catalog[self.NEXT_TABLE_ID_ATTR])
tables.append(self.TABLES_ASSIGN.join([name, str(id)]))
self.dom.put_attributes(self.CATALOG_ITEM,
{ self.TABLES_ATTR : self.TABLES_DELIM.join(tables),
self.NEXT_TABLE_ID_ATTR : str(id + 1) })
self.dom.put_attributes(str(id), { })
return Table(id, name)
def del_table(self, table):
assert table is not None
catalog = self.dom.get_item(self.CATALOG_ITEM)
tables = self.get_tables_assign()
for i in range(len(tables)):
tname, tid = tables[i].split(self.TABLES_ASSIGN)
tid = int(tid)
if tname == table.get_name():
del tables[i]
break
else:
assert False
self.dom.put_attributes(self.CATALOG_ITEM,
{ self.TABLES_ATTR : self.TABLES_DELIM.join(tables) })
if table.get_id() == int(catalog[self.NEXT_TABLE_ID_ATTR]) - 1:
self.dom.put_attributes(self.CATALOG_ITEM,
{ self.NEXT_TABLE_ID_ATTR : str(table.get_id()) })
t = self.dom.get_item(str(table.get_id()))
assert t is not None
self.dom.delete_item(t)