def graphs(request, project=""):
####
#Load any signals provided in the page
####
signals = []
timeRanges = utils.get_time_ranges()
for s in SIGNALS:
if s in request.POST:
signals.append( { 'value':urllib.unquote( request.POST[s] ),
'name':s } )
###
#Get reference data
###
cacheKey = str(project) + '_reference_data'
jsonData = '{}'
mc = memcache.Client([settings.DATAZILLA_MEMCACHED], debug=0)
compressedJsonData = mc.get(cacheKey)
timeKey = 'days_30'
##reference data found in the cache: decompress##
if compressedJsonData:
jsonData = zlib.decompress( compressedJsonData )
else:
####
#reference data has not been cached:
#serialize, compress, and cache
####
dm = DatazillaModel(project)
refData = dm.getTestReferenceData()
dm.disconnect()
refData['time_ranges'] = timeRanges
jsonData = json.dumps(refData)
mc.set(str(project) + '_reference_data', zlib.compress( jsonData ) )
data = { 'time_key':timeKey,
'reference_json':jsonData,
'signals':signals }
####
#Caller has provided the view parent of the signals, load in page.
#This occurs when a data view is in its Pane form and is detached
#to exist on it's own page.
####
parentIndexKey = 'dv_parent_dview_index'
if parentIndexKey in request.POST:
data[parentIndexKey] = request.POST[parentIndexKey]
return render_to_response('graphs.views.html', data)
def handle(self, *args, **options):
source = options.get('source')
target = options.get('target')
records = int( options.get('records') )
start = int( options.get('start') )
dm_source = DatazillaModel(source)
dm_target = DatazillaModel(target)
data_iter = dm_source.getAllTestData(start)
iterations = 0
for d in data_iter:
for data in d:
deserialized_data = json.loads( data['data'] )
if options['debug']:
self.stdout.write(data['data'] + "\n")
else:
if options['show']:
self.stdout.write(data['data'] + "\n")
dm_target.loadTestData(deserialized_data, data['data'])
iterations += 1
if iterations == records:
dm_source.disconnect()
dm_target.disconnect()
return
def dataview(request, project="", method=""):
procPath = "graphs.views."
##Full proc name including base path in json file##
fullProcPath = "%s%s" % (procPath, method)
if settings.DEBUG:
###
#Write IP address and datetime to log
###
print "Client IP:%s" % (request.META['REMOTE_ADDR'])
print "Request Datetime:%s" % (str(datetime.datetime.now()))
json = ""
if method in DATAVIEW_ADAPTERS:
dm = DatazillaModel(project)
if 'adapter' in DATAVIEW_ADAPTERS[method]:
json = DATAVIEW_ADAPTERS[method]['adapter'](project,
method,
request,
dm)
else:
if 'fields' in DATAVIEW_ADAPTERS[method]:
fields = []
for f in DATAVIEW_ADAPTERS[method]['fields']:
if f in request.POST:
fields.append( dm.dhub.escapeString( request.POST[f] ) )
elif f in request.GET:
fields.append( dm.dhub.escapeString( request.GET[f] ) )
if len(fields) == len(DATAVIEW_ADAPTERS[method]['fields']):
json = dm.dhub.execute(proc=fullProcPath,
debug_show=settings.DEBUG,
placeholders=fields,
return_type='table_json')
else:
json = '{ "error":"%s fields required, %s provided" }' % (str(len(DATAVIEW_ADAPTERS[method]['fields'])),
str(len(fields)))
else:
json = dm.dhub.execute(proc=fullProcPath,
debug_show=settings.DEBUG,
return_type='table_json')
dm.disconnect();
else:
json = '{ "error":"Data view name %s not recognized" }' % method
return HttpResponse(json, mimetype=APP_JS)
def setTestData(request):
jsonData = '{"error":"No POST data found"}'
if 'data' in request.POST:
jsonData = request.POST['data']
unquotedJsonData = urllib.unquote(jsonData)
data = json.loads( unquotedJsonData )
dm = DatazillaModel(project)
dm.loadTestData( data, unquotedJsonData )
dm.disconnect()
jsonData = json.dumps( { 'loaded_test_pages':len(data['results']) } )
return HttpResponse(jsonData, mimetype=APP_JS)
def cacheTestSummaries(project):
gm = DatazillaModel(project)
dataIter = gm.getAllSummaryCache()
mc = memcache.Client([settings.DATAZILLA_MEMCACHED], debug=0)
for d in dataIter:
for data in d:
key = utils.get_cache_key(
project,
data['item_id'],
data['item_data'],
)
rv = mc.set(key, zlib.compress( data['value'] ))
if not rv:
sys.stderr.write("ERROR: Failed to store object in memcache: %s, %s\n" % ( str(data['item_id']), data['item_data'] ) )
gm.disconnect()
def loadTestCollection(project):
gm = DatazillaModel(project)
products = gm.getProducts('id')
for productName in products:
if products[ productName ]['product'] and \
products[ productName ]['version'] and \
products[ productName ]['branch']:
name = "%s %s %s" % (products[ productName ]['product'],
products[ productName ]['version'],
products[ productName ]['branch'])
id = gm.set_data('set_test_collection', [ name, "", name ])
gm.set_data('set_test_collection_map', [ id, products[ productName ]['id'] ])
gm.disconnect()
def buildTestSummaries(project):
gm = DatazillaModel(project)
timeRanges = utils.get_time_ranges()
products = gm.getProducts()
for productName in products:
for tr in ['days_7', 'days_30']:
table = gm.getTestRunSummary(str( timeRanges[tr]['start']),
str( timeRanges[tr]['stop']),
[ products[ productName ] ],
[],
[])
jsonData = json.dumps( table )
gm.setSummaryCache( products[ productName ], tr, jsonData )
gm.disconnect()
def setUp(self):
self.gm = DatazillaModel("talos")
self.testData = """{"test_machine": {"name": "qm-pxp01", "os": "linux", "osversion": "Ubuntu 11.10", "platform": "x86_64"}, "test_build": {"name": "Firefox", "version": "13.0a1", "revision": "785345035a3b", "branch": "", "id": "20120228122102"}, "testrun": {"date": "1330454755", "suite": "Talos tp5r", "options": {"rss": true, "tpchrome": true, "tpmozafterpaint": false, "tpcycles": "3", "tppagecycles": 1, "tprender": false, "tpdelay": "", "responsiveness": false, "shutdown": true}}, "results": {"icanhascheezburger.com": [419.0, 424.0, 422.0], "ucoz.ru": [153.0, 127.0, 116.0], "alipay.com": [1704.0, 1054.0, 1097.0], "cgi.ebay.com": [373.0, 374.0, 346.0], "tmall.com": [432.0, 322.0, 335.0], "wp.pl": [265.0, 214.0, 249.0], "thesartorialist.blogspot.com": [739.0, 782.0, 792.0], "myparentswereawesome.tumblr.com": [182.0, 206.0, 211.0], "xunlei.com": [933.0, 909.0, 894.0], "huffingtonpost.com": [1080.0, 1077.0, 1064.0], "baidu.com": [1058.0, 66.0, 63.0], "reuters.com": [376.0, 299.0, 302.0], "chemistry.about.com": [224.0, 136.0, 143.0], "aljazeera.net": [333.0, 334.0, 333.0], "people.com.cn": [540.0, 468.0, 468.0], "naver.com": [387.0, 386.0, 381.0], "cnn.com": [1076.0, 966.0, 923.0], "imgur.com": [346.0, 230.0, 221.0], "guardian.co.uk": [409.0, 360.0, 390.0], "ask.com": [36.0, 24.0, 30.0], "hatena.ne.jp": [262.0, 215.0, 215.0], "hao123.com": [391.0, 108.0, 104.0], "tudou.com": [357.0, 307.0, 310.0], "rambler.ru": [279.0, 242.0, 255.0], "ifeng.com": [595.0, 641.0, 450.0], "w3schools.com": [139.0, 128.0, 115.0], "cakewrecks.blogspot.com": [280.0, 303.0, 288.0], "media.photobucket.com": [22.0, 28.0, 28.0], "noimpactman.typepad.com": [373.0, 371.0, 340.0], "reddit.com": [180.0, 213.0, 240.0], "163.com": [666.0, 587.0, 626.0], "dailymotion.com": [257.0, 271.0, 250.0], "wsj.com": [617.0, 558.0, 585.0], "whois.domaintools.com": [164.0, 83.0, 103.0], "terra.com.br": [367.0, 371.0, 377.0], "vkontakte.ru": [103.0, 83.0, 88.0], "seesaa.net": [266.0, 245.0, 228.0], "xinhuanet.com": [854.0, 865.0, 755.0], "csdn.net": [229.0, 197.0, 165.0], "slideshare.net": [218.0, 204.0, 169.0], "icious.com": [186.0, 169.0, 177.0], "bbc.co.uk": [320.0, 783.0, 320.0], "digg.com": [213.0, 195.0, 186.0], "orange.fr": [109.0, 117.0, 100.0], "spiegel.de": [444.0, 395.0, 440.0], "gmx.net": [266.0, 244.0, 240.0], "twitter.com": [216.0, 207.0, 196.0], "goo.ne.jp": [143.0, 122.0, 155.0], "yelp.com": [462.0, 410.0, 405.0], "msn.com": [266.0, 268.0, 248.0], "foxnews.com": [450.0, 417.0, 379.0], "ameblo.jp": [144.0, 123.0, 122.0], "amazon.com": [874.0, 745.0, 763.0], "taringa.net": [247.0, 232.0, 235.0], "mail.ru": [329.0, 295.0, 292.0], "ezinearticles.com": [154.0, 132.0, 163.0], "en.wikipedia.org": [437.0, 438.0, 412.0], "etsy.com": [224.0, 231.0, 236.0], "douban.com": [247.0, 209.0, 195.0], "mozilla.com": [406.0, 371.0, 378.0], "filestube.com": [178.0, 183.0, 183.0], "chinaz.com": [270.0, 224.0, 237.0], "w3.org": [125.0, 120.0, 126.0], "mashable.com": [591.0, 541.0, 538.0], "telegraph.co.uk": [328.0, 299.0, 286.0], "yahoo.co.jp": [93.0, 69.0, 86.0], "dailymail.co.uk": [1424.0, 1246.0, 1219.0], "page.renren.com": [241.0, 207.0, 219.0], "myspace.com": [683.0, 561.0, 555.0], "56.com": [789.0, 705.0, 739.0], "cnet.com": [431.0, 395.0, 404.0], "alibaba.com": [103.0, 95.0, 105.0], "yandex.ru": [245.0, 237.0, 244.0], "beatonna.livejournal.com": [312.0, 402.0, 391.0], "indiatimes.com": [295.0, 267.0, 300.0], "homeway.com.cn": [564.0, 473.0, 481.0], "globo.com": [848.0, 796.0, 887.0], "zol.com.cn": [652.0, 572.0, 580.0], "nicovideo.jp": [212.0, 203.0, 214.0], "youtube.com": [354.0, 274.0, 319.0], "sohu.com": [642.0, 566.0, 592.0], "rakuten.co.jp": [490.0, 471.0, 356.0], "stackoverflow.com": [319.0, 294.0, 323.0], "deviantart.com": [288.0, 258.0, 258.0], "store.apple.com": [592.0, 447.0, 452.0], "linkedin.com": [31.0, 27.0, 27.0], "php.net": [88.0, 86.0, 80.0], "thepiratebay.org": [206.0, 182.0, 189.0], "bing.com": [68.0, 65.0, 64.0], "facebook.com": [82.0, 77.0, 75.0], "bild.de": [1576.0, 1425.0, 1410.0], "ehow.com": [180.0, 171.0, 180.0], "torrentz.eu": [1050.0, 110.0, 113.0], "uol.com.br": [461.0, 453.0, 451.0], "blogfa.com": [92.0, 82.0, 73.0], "repubblica.it": [451.0, 368.0, 395.0], "web.de": [274.0, 230.0, 235.0], "youku.com": [580.0, 530.0, 530.0], "imdb.com": [586.0, 526.0, 503.0], "google.com": [89.0, 82.0, 83.0]}, "results_aux": {"main_rss": ["72122368", "89206784", "90710016", "93384704", "98676736", "102776832", "104378368", "107352064", "118648832", "116051968", "114241536", "119185408", "111882240", "115748864", "136167424", "133001216", "135909376", "141484032", "136798208", "136601600", "135417856", "152686592", "143863808", "140906496", "144883712", "142585856", "141848576", "139915264", "141393920", "136826880", "135618560", "139444224", "140664832", "138887168", "135860224", "137482240", "134184960", "135983104", "136122368", "138465280", "140492800", "142610432", "142323712", "141565952", "140443648", "139108352", "136720384", "135901184", "135589888", "140107776", "136724480", "136388608", "135663616", "134983680", "135331840", "132050944", "132956160", "133623808", "140271616", "138436608", "138723328", "139350016", "134299648", "134926336", "134578176", "142462976", "148357120", "172298240", "172904448", "167161856", "165482496", "157765632", "151285760", "149712896", "150904832", "152748032", "154136576", "161161216", "158474240", "153616384", "153759744", "159219712", "159309824", "159223808", "159318016", "157437952", "162385920", "161898496", "163717120", "163323904", "160092160", "155025408", "156364800", "154722304", "153915392", "154660864", "152289280", "159399936", "152625152", "151363584", "162496512", "162648064", "164470784", "155590656", "154185728", "152379392", "150921216", "151949312", "152285184", "159297536", "156372992", "156557312", "161239040", "155205632", "158298112", "167124992", "162471936", "161673216", "164835328", "161984512", "160825344", "158281728", "164982784", "158330880", "155975680", "158412800", "162308096", "161017856", "159277056", "159666176", "157941760", "158040064", "157437952", "159182848", "157253632", "153051136", "155262976", "151986176", "153169920", "152662016", "153841664", "158158848", "157790208", "157347840", "157270016", "153976832", "152989696", "152559616", "151842816", "153468928", "155406336", "154587136", "151343104", "150654976", "147668992", "149622784", "147365888", "149688320", "148840448", "153718784", "151138304", "151240704", "152625152", "149356544", "147849216", "149094400", "155758592", "157757440", "173686784", "173670400", "169566208", "168361984", "159285248", "155049984", "151773184", "153034752", "156340224", "157769728", "163049472", "160309248", "156725248", "156200960", "161132544", "158035968", "159838208", "159248384", "156876800", "163905536", "164196352", "163553280", "162721792", "159043584", "155623424", "157310976", "154398720", "152690688", "152023040", "150384640", "155734016", "150310912", "148156416", "159916032", "158863360", "162074624", "150343680", "150003712", "149815296", "149909504", "150462464", "151019520", "154513408", "153325568", "151511040", "158121984", "151826432", "155500544", "161931264", "157065216", "157601792", "159711232", "157159424", "153587712", "152948736", "159780864", "151695360", "147214336", "151302144", "152567808", "157495296", "157966336", "157097984", "153907200", "150777856", "151244800", "152883200", "151523328", "148725760", "149188608", "146071552", "147095552", "147664896", "149254144", "150831104", "151146496", "152391680", "151142400", "149803008", "146571264", "146280448", "144437248", "146022400", "147415040", "148787200", "150003712", "147161088", "145874944", "148062208", "146587648", "146239488", "146448384", "149618688", "146731008", "146817024", "148029440", "143884288", "143904768", "144338944", "150528000", "153956352", "170328064", "169480192", "166031360", "164032512", "156352512", "152584192", "149594112", "151015424", "152793088", "152854528", "158572544", "155385856", "151617536", "151322624", "156491776", "156041216", "155545600", "157372416", "152903680", "157818880", "158097408", "159756288", "159305728", "156852224", "155230208", "154509312", "151814144", "150151168", "150573056", "149635072", "153280512", "151470080", "150413312", "159612928"], "shutdown": [904]}}"""