def set_test_data(request, project=""):
"""
Post a JSON blob of data for the specified project.
Store the JSON in the objectstore where it will be held for
later processing.
"""
#####
#This conditional provides backwords compatibility with
#the talos production environment. It should
#be removed after the production environment
#is uniformaly using the new url format.
####
if project == 'views':
project = 'talos'
# default to bad request if the JSON is malformed or not present
status = 400
try:
json_data = request.POST['data']
except KeyError:
result = {"status":"No POST data found"}
else:
unquoted_json_data = urllib.unquote(json_data)
error = None
try:
json.loads( unquoted_json_data )
except ValueError as e:
error = "Malformed JSON: {0}".format(e.message)
result = {"status": "Malformed JSON", "message": error}
else:
result = {
"status": "well-formed JSON stored",
"size": len(unquoted_json_data),
}
try:
dm = PerformanceTestModel(project)
dm.store_test_data(unquoted_json_data, error)
dm.disconnect()
except Exception as e:
status = 500
result = {"status": "Unknown error", "message": str(e)}
else:
if not error:
status = 200
return HttpResponse(json.dumps(result), mimetype=APP_JS, status=status)
def set_test_data(request, project=""):
"""
Post a JSON blob of data for the specified project.
Store the JSON in the objectstore where it will be held for
later processing.
"""
#####
#This conditional provides backwords compatibility with
#the talos production environment. It should
#be removed after the production environment
#is uniformaly using the new url format.
####
if project == 'views':
project = 'talos'
# default to bad request if the JSON is malformed or not present
status = 400
try:
json_data = request.POST['data']
except KeyError:
result = {"status": "No POST data found"}
else:
unquoted_json_data = urllib.unquote(json_data)
error = None
deserialized_json = {}
try:
deserialized_json = json.loads(unquoted_json_data)
except ValueError as e:
error = "Malformed JSON: {0}".format(e.message)
result = {"status": "Malformed JSON", "message": error}
else:
result = {
"status": "well-formed JSON stored",
"size": len(unquoted_json_data),
}
try:
dm = PerformanceTestModel(project)
dm.pre_process_data(unquoted_json_data, deserialized_json)
id = dm.store_test_data(unquoted_json_data, error)
dm.disconnect()
except Exception as e:
status = 500
result = {"status": "Unknown error", "message": str(e)}
else:
location = "/{0}/refdata/objectstore/json_blob/{1}".format(
project, str(id))
result['url'] = request.build_absolute_uri(location)
if not error:
status = 200
return HttpResponse(json.dumps(result), mimetype=APP_JS, status=status)
def handle(self, *args, **options):
""" Transfer data to a development project based on the args value. """
host = options.get("host")
dev_project = options.get("dev_project")
prod_project = options.get("prod_project")
branch = options.get("branch")
days_ago = options.get("days_ago")
logfile = options.get("logfile")
if not host:
self.println("You must supply a host name to retrieve data from " +
"--host hostname")
return
if not dev_project:
self.println("You must supply a dev_project name to load data in.")
return
if not branch:
self.println("You must supply a branch name to retrieve data for.")
return
#Set timeout so we don't hang
timeout = 120
socket.setdefaulttimeout(timeout)
revisions_uri = 'refdata/pushlog/list'
params = 'days_ago={0}&branches={1}'.format(days_ago, branch)
url = "https://{0}/{1}?{2}".format(host, revisions_uri, params)
json_data = ""
#Retrieve revisions to iterate over
try:
response = urllib.urlopen(url)
json_data = response.read()
except socket.timeout:
self.stdout.write( "URL: {0}\nTimedout {1} seconds\n".format(
url, timeout
) )
sys.exit(0)
data = json.loads(json_data)
all_keys = data.keys()
all_keys.sort()
##Keep track of revision already loaded##
file_obj = open(logfile, 'w+')
revisions_seen = set()
for line in file_obj.readlines():
revisions_seen.add(line.strip())
revisions = []
for key in all_keys:
for revision in data[key]['revisions']:
if revision not in revisions_seen:
revisions.append(revision)
dm = PerformanceTestModel(dev_project)
for revision in revisions:
rawdata_uri = '{0}/testdata/raw/{1}/{2}/'.format(
prod_project, branch, revision
)
rawdata_url = "https://{0}/{1}".format(host, rawdata_uri)
raw_json_data = ""
try:
rawdata_response = urllib.urlopen(rawdata_url)
raw_json_data = rawdata_response.read()
except socket.timeout:
self.stdout.write( "URL: {0}\nTimedout {1} seconds\n".format(
rawdata_url, timeout) )
sys.exit(0)
test_object_data = json.loads(raw_json_data)
for test_object in test_object_data:
id = dm.store_test_data( json.dumps(test_object), "" )
self.stdout.write( "Revision:{0} Id:{1}\n".format(revision, str(id)))
#Record the revision as loaded
file_obj.write(revision + "\n")
file_obj.close()
dm.disconnect()
def handle(self, *args, **options):
""" Transfer data to a development project based on the args value. """
host = options.get("host")
dev_project = options.get("dev_project")
prod_project = options.get("prod_project")
branch = options.get("branch")
days_ago = options.get("days_ago")
logfile = options.get("logfile")
if not host:
self.println("You must supply a host name to retrieve data from " +
"--host hostname")
return
if not dev_project:
self.println("You must supply a dev_project name to load data in.")
return
if not branch:
self.println("You must supply a branch name to retrieve data for.")
return
#Set timeout so we don't hang
timeout = 120
socket.setdefaulttimeout(timeout)
revisions_uri = 'refdata/pushlog/list'
params = 'days_ago={0}&branches={1}'.format(days_ago, branch)
url = "https://{0}/{1}?{2}".format(host, revisions_uri, params)
json_data = ""
#Retrieve revisions to iterate over
try:
response = urllib.urlopen(url)
json_data = response.read()
except socket.timeout:
self.stdout.write("URL: {0}\nTimedout {1} seconds\n".format(
url, timeout))
sys.exit(0)
data = json.loads(json_data)
all_keys = data.keys()
all_keys.sort()
##Keep track of revision already loaded##
file_obj = open(logfile, 'w+')
revisions_seen = set()
for line in file_obj.readlines():
revisions_seen.add(line.strip())
revisions = []
for key in all_keys:
for revision in data[key]['revisions']:
if revision not in revisions_seen:
revisions.append(revision)
dm = PerformanceTestModel(dev_project)
for revision in revisions:
rawdata_uri = '{0}/testdata/raw/{1}/{2}/'.format(
prod_project, branch, revision)
rawdata_url = "https://{0}/{1}".format(host, rawdata_uri)
raw_json_data = ""
try:
rawdata_response = urllib.urlopen(rawdata_url)
raw_json_data = rawdata_response.read()
except socket.timeout:
self.stdout.write("URL: {0}\nTimedout {1} seconds\n".format(
rawdata_url, timeout))
sys.exit(0)
test_object_data = json.loads(raw_json_data)
for test_object in test_object_data:
id = dm.store_test_data(json.dumps(test_object), "")
self.stdout.write("Revision:{0} Id:{1}\n".format(
revision, str(id)))
#Record the revision as loaded
file_obj.write(revision + "\n")
file_obj.close()
dm.disconnect()
