def handle_project(self, project, **options):
debug = options.get("debug", None)
max_iterations = int(options.get("iterations", 50))
ptm = PerformanceTestModel(project)
sql_targets = {}
cycle_iterations = max_iterations
while cycle_iterations > 0:
sql_targets = ptm.cycle_data(sql_targets)
if debug:
print "Iterations: {0}".format(str(cycle_iterations))
print "sql_targets"
print sql_targets
cycle_iterations -= 1
if sql_targets['total_count'] == 0:
cycle_iterations = 0
ptm.disconnect()
def graphs(request, project=""):
####
#Load any signals provided in the page
####
signals = []
time_ranges = utils.get_time_ranges()
for s in SIGNALS:
if s in request.POST:
signals.append( { 'value':urllib.unquote( request.POST[s] ),
'name':s } )
###
#Get reference data
###
ptm = PerformanceTestModel(project)
json_data = ptm.get_test_reference_data()
ptm.disconnect()
time_key = 'days_30'
data = { 'time_key':time_key,
'reference_json':json_data,
'signals':signals }
####
#Caller has provided the view parent of the signals, load in page.
#This occurs when a data view is in its Pane form and is detached
#to exist on it's own page.
####
parent_index_key = 'dv_parent_dview_index'
if parent_index_key in request.POST:
data[parent_index_key] = request.POST[parent_index_key]
return render_to_response('graphs.views.html', data)
def load_test_collection(project):
ptm = PerformanceTestModel(project)
products = ptm.get_products('id')
product_names = {}
for id in products:
if products[ id ]['product'] and \
products[ id ]['version'] and \
products[ id ]['branch']:
name = get_test_collection_name(
products[ id ]['product'],
products[ id ]['version'],
products[ id ]['branch']
)
product_names[name] = id
test_collection_names = ptm.get_test_collection_set()
new_name_set = set(
product_names.keys()
).difference( test_collection_names )
for new_name in new_name_set:
id = ptm.set_test_collection(new_name, "")
ptm.set_test_collection_map(id, product_names[ new_name ])
ptm.cache_ref_data()
ptm.disconnect()
def load_test_collection(project):
ptm = PerformanceTestModel(project)
products = ptm.get_products('id')
product_names = {}
for id in products:
if products[ id ]['product'] and \
products[ id ]['version'] and \
products[ id ]['branch']:
name = get_test_collection_name(products[id]['product'],
products[id]['version'],
products[id]['branch'])
product_names[name] = id
test_collection_names = ptm.get_test_collection_set()
new_name_set = set(product_names.keys()).difference(test_collection_names)
for new_name in new_name_set:
id = ptm.set_test_collection(new_name, "")
ptm.set_test_collection_map(id, product_names[new_name])
ptm.cache_ref_data()
ptm.disconnect()
def cache_test_summaries(project):
ptm = PerformanceTestModel(project)
###
#New reference data could be found in the cached data
#summary structures. Update the reference data cached
#every time the sumary data is cached.
###
ptm.cache_ref_data()
ptm.cache_default_project()
data_iter = ptm.get_all_summary_cache()
key_test = []
for d in data_iter:
for data in d:
key = utils.get_summary_cache_key(
project,
data['item_id'],
data['item_data'],
)
rv = cache.set(key, zlib.compress(data['value']))
if not rv:
msg = "ERROR: Failed to store object in memcache: %s, %s\n" % \
( str(data['item_id']), data['item_data'] )
sys.stderr.write(msg)
ptm.disconnect()
def handle_project(self, project, **options):
debug = options.get("debug", None)
max_iterations = int(options.get("iterations", 50))
ptm = PerformanceTestModel(project)
sql_targets = {}
cycle_iterations = max_iterations
while cycle_iterations > 0:
sql_targets = ptm.cycle_data(sql_targets)
if debug:
print "Iterations: {0}".format(str(cycle_iterations))
print "sql_targets"
print sql_targets
cycle_iterations -= 1
if sql_targets['total_count'] == 0:
cycle_iterations = 0
ptm.disconnect()
def set_test_data(request, project=""):
"""
Post a JSON blob of data for the specified project.
Store the JSON in the objectstore where it will be held for
later processing.
"""
#####
#This conditional provides backwords compatibility with
#the talos production environment. It should
#be removed after the production environment
#is uniformaly using the new url format.
####
if project == 'views':
project = 'talos'
# default to bad request if the JSON is malformed or not present
status = 400
try:
json_data = request.POST['data']
except KeyError:
result = {"status":"No POST data found"}
else:
unquoted_json_data = urllib.unquote(json_data)
error = None
try:
json.loads( unquoted_json_data )
except ValueError as e:
error = "Malformed JSON: {0}".format(e.message)
result = {"status": "Malformed JSON", "message": error}
else:
result = {
"status": "well-formed JSON stored",
"size": len(unquoted_json_data),
}
try:
dm = PerformanceTestModel(project)
id = dm.store_test_data(unquoted_json_data, error)
dm.disconnect()
except Exception as e:
status = 500
result = {"status": "Unknown error", "message": str(e)}
else:
location = "/{0}/refdata/objectstore/json_blob/{1}".format(
project, str(id)
)
result['url'] = request.build_absolute_uri(location)
if not error:
status = 200
return HttpResponse(json.dumps(result), mimetype=APP_JS, status=status)
def cache_test_summaries(project):
ptm = PerformanceTestModel(project)
###
#New reference data could be found in the cached data
#summary structures. Update the reference data cached
#every time the sumary data is cached.
###
ptm.cache_ref_data()
ptm.cache_default_project()
data_iter = ptm.get_all_summary_cache()
for d in data_iter:
for data in d:
key = utils.get_summary_cache_key(
project,
data['item_id'],
data['item_data'],
)
rv = cache.set(key, zlib.compress( data['value'] ))
if not rv:
msg = "ERROR: Failed to store object in memcache: %s, %s\n" % \
( str(data['item_id']), data['item_data'] )
sys.stderr.write(msg)
ptm.disconnect()
def handle_project(self, project, options):
self.stdout.write("Processing project {0}\n".format(project))
loadlimit = int(options.get("loadlimit", 1))
dm = PerformanceTestModel(project)
dm.process_objects(loadlimit)
dm.disconnect()
def dataview(request, project="", method=""):
proc_path = "perftest.views."
##Full proc name including base path in json file##
full_proc_path = "%s%s" % (proc_path, method)
if settings.DEBUG:
###
#Write IP address and datetime to log
###
print "Client IP:%s" % (request.META['REMOTE_ADDR'])
print "Request Datetime:%s" % (str(datetime.datetime.now()))
json = ""
if method in DATAVIEW_ADAPTERS:
dm = PerformanceTestModel(project)
pt_dhub = dm.sources["perftest"].dhub
if 'adapter' in DATAVIEW_ADAPTERS[method]:
json = DATAVIEW_ADAPTERS[method]['adapter'](project,
method,
request,
dm)
else:
if 'fields' in DATAVIEW_ADAPTERS[method]:
fields = []
for f in DATAVIEW_ADAPTERS[method]['fields']:
if f in request.GET:
fields.append( int( request.GET[f] ) )
if len(fields) == len(DATAVIEW_ADAPTERS[method]['fields']):
json = pt_dhub.execute(proc=full_proc_path,
debug_show=settings.DEBUG,
placeholders=fields,
return_type='table_json')
else:
json = '{ "error":"{0} fields required, {1} provided" }'.format(
(str(len(DATAVIEW_ADAPTERS[method]['fields'])),
str(len(fields))) )
else:
json = pt_dhub.execute(proc=full_proc_path,
debug_show=settings.DEBUG,
return_type='table_json')
dm.disconnect();
else:
json = '{ "error":"Data view name %s not recognized" }' % method
return HttpResponse(json, mimetype=APP_JS)
def dataview(request, project="", method=""):
proc_path = "perftest.views."
##Full proc name including base path in json file##
full_proc_path = "%s%s" % (proc_path, method)
if settings.DEBUG:
###
#Write IP address and datetime to log
###
print "Client IP:%s" % (request.META['REMOTE_ADDR'])
print "Request Datetime:%s" % (str(datetime.datetime.now()))
json = ""
if method in DATAVIEW_ADAPTERS:
dm = PerformanceTestModel(project)
pt_dhub = dm.sources["perftest"].dhub
if 'adapter' in DATAVIEW_ADAPTERS[method]:
json = DATAVIEW_ADAPTERS[method]['adapter'](project, method,
request, dm)
else:
if 'fields' in DATAVIEW_ADAPTERS[method]:
fields = []
for f in DATAVIEW_ADAPTERS[method]['fields']:
if f in request.GET:
fields.append(int(request.GET[f]))
if len(fields) == len(DATAVIEW_ADAPTERS[method]['fields']):
json = pt_dhub.execute(proc=full_proc_path,
debug_show=settings.DEBUG,
placeholders=fields,
return_type='table_json')
else:
json = '{ "error":"{0} fields required, {1} provided" }'.format(
(str(len(DATAVIEW_ADAPTERS[method]['fields'])),
str(len(fields))))
else:
json = pt_dhub.execute(proc=full_proc_path,
debug_show=settings.DEBUG,
return_type='table_json')
dm.disconnect()
else:
json = '{ "error":"Data view name %s not recognized" }' % method
return HttpResponse(json, mimetype=APP_JS)
def handle_project(self, project, **options):
self.stdout.write("Processing project {0}\n".format(project))
pushlog_project = options.get("pushlog_project", 'pushlog')
loadlimit = int(options.get("loadlimit", 1))
debug = options.get("debug", None)
test_run_ids = []
ptm = PerformanceTestModel(project)
test_run_ids = ptm.process_objects(loadlimit)
ptm.disconnect()
"""
metrics_exclude_projects = set(['b2g', 'games', 'jetperf', 'marketapps', 'microperf', 'stoneridge', 'test', 'webpagetest'])
if project not in metrics_exclude_projects:
#minimum required number of replicates for
#metrics processing
replicate_min = 5
compute_test_run_metrics(
project, pushlog_project, debug, replicate_min, test_run_ids
)
"""
mtm = MetricsTestModel(project)
revisions_without_push_data = mtm.load_test_data_all_dimensions(
test_run_ids)
if revisions_without_push_data:
revision_nodes = {}
plm = PushLogModel(pushlog_project)
for revision in revisions_without_push_data:
node = plm.get_node_from_revision(
revision, revisions_without_push_data[revision])
revision_nodes[revision] = node
plm.disconnect()
mtm.set_push_data_all_dimensions(revision_nodes)
mtm.disconnect()
def build_test_summaries(project):
ptm = PerformanceTestModel(project)
time_ranges = utils.get_time_ranges()
products = ptm.get_products()
for product_name in products:
for tr in ['days_7', 'days_30']:
table = ptm.get_test_run_summary(str(time_ranges[tr]['start']),
str(time_ranges[tr]['stop']),
[products[product_name]], [], [])
json_data = json.dumps(table)
ptm.set_summary_cache(products[product_name], tr, json_data)
ptm.disconnect()
def handle_project(self, project, **options):
self.stdout.write("Processing project {0}\n".format(project))
pushlog_project = options.get("pushlog_project", 'pushlog')
loadlimit = int(options.get("loadlimit", 1))
debug = options.get("debug", None)
test_run_ids = []
ptm = PerformanceTestModel(project)
test_run_ids = ptm.process_objects(loadlimit)
ptm.disconnect()
metrics_exclude_projects = set(['b2g', 'stoneridge'])
if project not in metrics_exclude_projects:
#minimum required number of replicates for
#metrics processing
replicate_min = 5
compute_test_run_metrics(
project, pushlog_project, debug, replicate_min, test_run_ids
)
def graphs(request, project=""):
####
#Load any signals provided in the page
####
signals = []
time_ranges = utils.get_time_ranges()
for s in SIGNALS:
if s in request.POST:
signals.append({
'value': urllib.unquote(request.POST[s]),
'name': s
})
###
#Get reference data
###
ptm = PerformanceTestModel(project)
json_data = ptm.get_test_reference_data()
ptm.disconnect()
time_key = 'days_30'
data = {
'time_key': time_key,
'reference_json': json_data,
'signals': signals
}
####
#Caller has provided the view parent of the signals, load in page.
#This occurs when a data view is in its Pane form and is detached
#to exist on it's own page.
####
parent_index_key = 'dv_parent_dview_index'
if parent_index_key in request.POST:
data[parent_index_key] = request.POST[parent_index_key]
return render_to_response('graphs.views.html', data)
def build_test_summaries(project):
ptm = PerformanceTestModel(project)
time_ranges = utils.get_time_ranges()
products = ptm.get_products()
for product_name in products:
for tr in ['days_7', 'days_30']:
table = ptm.get_test_run_summary(str( time_ranges[tr]['start']),
str( time_ranges[tr]['stop']),
[ products[ product_name ] ],
[],
[])
json_data = json.dumps( table )
ptm.set_summary_cache( products[ product_name ], tr, json_data )
ptm.disconnect()
def handle(self, *args, **options):
""" Transfer data to a development project based on the args value. """
host = options.get("host")
dev_project = options.get("dev_project")
prod_project = options.get("prod_project")
branch = options.get("branch")
days_ago = options.get("days_ago")
logfile = options.get("logfile")
if not host:
self.println("You must supply a host name to retrieve data from " +
"--host hostname")
return
if not dev_project:
self.println("You must supply a dev_project name to load data in.")
return
if not branch:
self.println("You must supply a branch name to retrieve data for.")
return
#Set timeout so we don't hang
timeout = 120
socket.setdefaulttimeout(timeout)
revisions_uri = 'refdata/pushlog/list'
params = 'days_ago={0}&branches={1}'.format(days_ago, branch)
url = "https://{0}/{1}?{2}".format(host, revisions_uri, params)
json_data = ""
#Retrieve revisions to iterate over
try:
response = urllib.urlopen(url)
json_data = response.read()
except socket.timeout:
self.stdout.write( "URL: {0}\nTimedout {1} seconds\n".format(
url, timeout
) )
sys.exit(0)
data = json.loads(json_data)
all_keys = data.keys()
all_keys.sort()
##Keep track of revision already loaded##
file_obj = open(logfile, 'w+')
revisions_seen = set()
for line in file_obj.readlines():
revisions_seen.add(line.strip())
revisions = []
for key in all_keys:
for revision in data[key]['revisions']:
if revision not in revisions_seen:
revisions.append(revision)
dm = PerformanceTestModel(dev_project)
for revision in revisions:
rawdata_uri = '{0}/testdata/raw/{1}/{2}/'.format(
prod_project, branch, revision
)
rawdata_url = "https://{0}/{1}".format(host, rawdata_uri)
raw_json_data = ""
try:
rawdata_response = urllib.urlopen(rawdata_url)
raw_json_data = rawdata_response.read()
except socket.timeout:
self.stdout.write( "URL: {0}\nTimedout {1} seconds\n".format(
rawdata_url, timeout) )
sys.exit(0)
test_object_data = json.loads(raw_json_data)
for test_object in test_object_data:
id = dm.store_test_data( json.dumps(test_object), "" )
self.stdout.write( "Revision:{0} Id:{1}\n".format(revision, str(id)))
#Record the revision as loaded
file_obj.write(revision + "\n")
file_obj.close()
dm.disconnect()
def set_test_data(request, project=""):
"""
Post a JSON blob of data for the specified project.
Store the JSON in the objectstore where it will be held for
later processing.
"""
#####
#This conditional provides backwords compatibility with
#the talos production environment. It should
#be removed after the production environment
#is uniformaly using the new url format.
####
if project == 'views':
project = 'talos'
# default to bad request if the JSON is malformed or not present
status = 400
try:
json_data = request.POST['data']
except KeyError:
result = {"status": "No POST data found"}
else:
unquoted_json_data = urllib.unquote(json_data)
error = None
deserialized_json = {}
try:
deserialized_json = json.loads(unquoted_json_data)
except ValueError as e:
error = "Malformed JSON: {0}".format(e.message)
result = {"status": "Malformed JSON", "message": error}
else:
result = {
"status": "well-formed JSON stored",
"size": len(unquoted_json_data),
}
try:
dm = PerformanceTestModel(project)
dm.pre_process_data(unquoted_json_data, deserialized_json)
id = dm.store_test_data(unquoted_json_data, error)
dm.disconnect()
except Exception as e:
status = 500
result = {"status": "Unknown error", "message": str(e)}
else:
location = "/{0}/refdata/objectstore/json_blob/{1}".format(
project, str(id))
result['url'] = request.build_absolute_uri(location)
if not error:
status = 200
return HttpResponse(json.dumps(result), mimetype=APP_JS, status=status)
def handle(self, *args, **options):
""" Transfer data to a development project based on the args value. """
host = options.get("host")
dev_project = options.get("dev_project")
prod_project = options.get("prod_project")
branch = options.get("branch")
days_ago = options.get("days_ago")
logfile = options.get("logfile")
if not host:
self.println("You must supply a host name to retrieve data from " +
"--host hostname")
return
if not dev_project:
self.println("You must supply a dev_project name to load data in.")
return
if not branch:
self.println("You must supply a branch name to retrieve data for.")
return
#Set timeout so we don't hang
timeout = 120
socket.setdefaulttimeout(timeout)
revisions_uri = 'refdata/pushlog/list'
params = 'days_ago={0}&branches={1}'.format(days_ago, branch)
url = "https://{0}/{1}?{2}".format(host, revisions_uri, params)
json_data = ""
#Retrieve revisions to iterate over
try:
response = urllib.urlopen(url)
json_data = response.read()
except socket.timeout:
self.stdout.write("URL: {0}\nTimedout {1} seconds\n".format(
url, timeout))
sys.exit(0)
data = json.loads(json_data)
all_keys = data.keys()
all_keys.sort()
##Keep track of revision already loaded##
file_obj = open(logfile, 'w+')
revisions_seen = set()
for line in file_obj.readlines():
revisions_seen.add(line.strip())
revisions = []
for key in all_keys:
for revision in data[key]['revisions']:
if revision not in revisions_seen:
revisions.append(revision)
dm = PerformanceTestModel(dev_project)
for revision in revisions:
rawdata_uri = '{0}/testdata/raw/{1}/{2}/'.format(
prod_project, branch, revision)
rawdata_url = "https://{0}/{1}".format(host, rawdata_uri)
raw_json_data = ""
try:
rawdata_response = urllib.urlopen(rawdata_url)
raw_json_data = rawdata_response.read()
except socket.timeout:
self.stdout.write("URL: {0}\nTimedout {1} seconds\n".format(
rawdata_url, timeout))
sys.exit(0)
test_object_data = json.loads(raw_json_data)
for test_object in test_object_data:
id = dm.store_test_data(json.dumps(test_object), "")
self.stdout.write("Revision:{0} Id:{1}\n".format(
revision, str(id)))
#Record the revision as loaded
file_obj.write(revision + "\n")
file_obj.close()
dm.disconnect()
