def post(self): file_params = self.request.POST.getall(PARAM_FILE) if not file_params: self.response.out.write("FAIL: missing upload file field.") return builder = self.request.get(PARAM_BUILDER) if not builder: self.response.out.write("FAIL: missing builder parameter.") return master = self.request.get(PARAM_MASTER) test_type = self.request.get(PARAM_TEST_TYPE) logging.debug( "Processing upload request, master: %s, builder: %s, test_type: %s.", master, builder, test_type) # There are two possible types of each file_params in the request: # one file item or a list of file items. # Normalize file_params to a file item list. files = [] logging.debug("test: %s, type:%s", file_params, type(file_params)) for item in file_params: if not isinstance(item, list) and not isinstance(item, tuple): item = [item] files.extend(item) errors = [] final_status_code = 200 for file in files: if file.filename == "incremental_results.json": status_string, status_code = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=False) elif file.filename == "times_ms.json": # We never look at historical times_ms.json files, so we can overwrite the existing one if it exists. status_string, status_code = TestFile.overwrite_or_add_file(master, builder, test_type, file.filename, file.value) else: status_string, status_code = TestFile.add_file(master, builder, test_type, file.filename, file.value) # FIXME: Upload full_results.json files for non-layout tests as well and stop supporting the # incremental_results.json file format. if status_code == 200 and file.filename == "full_results.json": status_string, status_code = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=True) if status_code == 200: logging.info(status_string) else: logging.error(status_string) errors.append(status_string) final_status_code = status_code if errors: messages = "FAIL: " + "; ".join(errors) self.response.set_status(final_status_code, messages) self.response.out.write(messages) else: self.response.set_status(200) self.response.out.write("OK")
def get(self): key = self.request.get(PARAM_KEY) builder = self.request.get(PARAM_BUILDER) test_type = self.request.get(PARAM_TEST_TYPE) name = self.request.get(PARAM_NAME) logging.debug( "Deleting File, builder: %s, test_type: %s, name: %s, key: %s.", builder, test_type, name, key) TestFile.delete_file(key, builder, test_type, name, 100) # Display file list after deleting the file. self.redirect("/testfile?builder=%s&testtype=%s&name=%s" % (builder, test_type, name))
def _get_file_list(self, master, builder, test_type, name, before, limit, callback_name=None): """Get and display a list of files that matches builder and file name. Args: builder: builder name test_type: type of the test name: file name """ files = TestFile.get_files( master, builder, test_type, name, before, load_data=False, limit=limit) if not files: logging.info("File not found, master: %s, builder: %s, test_type: %s, name: %s.", master, builder, test_type, name) self.response.out.write("File not found") return template_values = { "admin": users.is_current_user_admin(), "master": master, "builder": builder, "test_type": test_type, "name": name, "files": files, } if callback_name: json = template.render("templates/showfilelist.jsonp", template_values) self._serve_json(_replace_jsonp_callback(json, callback_name), files[0].date) return self.response.out.write(template.render("templates/showfilelist.html", template_values))
def _get_file_list(self, master, builder, test_type, name, callback_name=None): """Get and display a list of files that matches builder and file name. Args: builder: builder name test_type: type of the test name: file name """ files = TestFile.get_files( master, builder, test_type, name, load_data=False, limit=100) if not files: logging.info("File not found, master: %s, builder: %s, test_type: %s, name: %s.", master, builder, test_type, name) self.response.out.write("File not found") return template_values = { "admin": users.is_current_user_admin(), "master": master, "builder": builder, "test_type": test_type, "name": name, "files": files, } if callback_name: json = template.render("templates/showfilelist.jsonp", template_values) self._serve_json(_replace_jsonp_callback(json, callback_name), files[0].date) return self.response.out.write(template.render("templates/showfilelist.html", template_values))
def _get_file_list(self, builder, test_type, name): """Get and display a list of files that matches builder and file name. Args: builder: builder name test_type: type of the test name: file name """ files = TestFile.get_files(builder, test_type, name, load_data=False, limit=100) if not files: logging.info( "File not found, builder: %s, test_type: %s, name: %s.", builder, test_type, name) self.response.out.write("File not found") return template_values = { "admin": users.is_current_user_admin(), "builder": builder, "test_type": test_type, "name": name, "files": files, } self.response.out.write( template.render("templates/showfilelist.html", template_values))
def _get_file_list(self, builder, test_type, name): """Get and display a list of files that matches builder and file name. Args: builder: builder name test_type: type of the test name: file name """ files = TestFile.get_files( builder, test_type, name, load_data=False, limit=100) if not files: logging.info("File not found, builder: %s, test_type: %s, name: %s.", builder, test_type, name) self.response.out.write("File not found") return template_values = { "admin": users.is_current_user_admin(), "builder": builder, "test_type": test_type, "name": name, "files": files, } self.response.out.write(template.render("templates/showfilelist.html", template_values))
def _get_file_content(self, master, builder, test_type, build_number, name): """Return content of the file that matches builder and file name. Args: builder: builder name test_type: type of the test build_number: build number, or 'latest' name: file name """ files = TestFile.get_files(master, builder, test_type, build_number, name, load_data=True, limit=1) if not files: logging.info( "File not found, master %s, builder: %s, test_type: %s, build_number: %s, name: %s.", master, builder, test_type, build_number, name) return None, None return files[0].data, files[0].date
def post(self): uploaded_files = self.get_uploads("file") if not uploaded_files: return self._upload_done([("Missing upload file field.")]) builder = self.request.get(PARAM_BUILDER) if not builder: for blob_info in uploaded_files: blob_info.delete() return self._upload_done([ ("Missing builder parameter in upload request.") ]) test_type = self.request.get(PARAM_TEST_TYPE) logging.debug("Processing upload request, builder: %s, test_type: %s.", builder, test_type) errors = [] for blob_info in uploaded_files: tf = TestFile.update_file(builder, test_type, blob_info) if not tf: errors.append( "Upload failed, builder: %s, test_type: %s, name: %s." % (builder, test_type, blob_info.filename)) blob_info.delete() return self._upload_done(errors)
def update(cls, builder, test_type, incremental): """Update datastore json file data by merging it with incremental json file. Args: builder: builder name. test_type: type of test results. incremental: incremental json file data to merge. Returns: TestFile object if update succeeds or None on failure. """ files = TestFile.get_files(builder, test_type, JSON_RESULTS_FILE) if files: file = files[0] new_results = cls.merge(builder, file.data, incremental) else: # Use the incremental data if there is no aggregated file to merge. file = TestFile() file.builder = builder file.test_type = test_type file.name = JSON_RESULTS_FILE new_results = incremental logging.info( "No existing json results, incremental json is saved.") if not new_results: return None if not file.save(new_results): return None return file
def post(self): file_params = self.request.POST.getall(PARAM_FILE) if not file_params: self.response.out.write("FAIL: missing upload file field.") return builder = self.request.get(PARAM_BUILDER) if not builder: self.response.out.write("FAIL: missing builder parameter.") return master = self.request.get(PARAM_MASTER) test_type = self.request.get(PARAM_TEST_TYPE) logging.debug( "Processing upload request, master: %s, builder: %s, test_type: %s.", master, builder, test_type) # There are two possible types of each file_params in the request: # one file item or a list of file items. # Normalize file_params to a file item list. files = [] logging.debug("test: %s, type:%s", file_params, type(file_params)) for item in file_params: if not isinstance(item, list) and not isinstance(item, tuple): item = [item] files.extend(item) errors = [] for file in files: if file.filename == "incremental_results.json": # FIXME: Remove this check once we stop uploading incremental_results.json files for layout tests. if test_type == "layout-tests": update_succeeded = True else: update_succeeded = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=False) else: update_succeeded = bool(TestFile.add_file(master, builder, test_type, file.filename, file.value)) # FIXME: Upload full_results.json files for non-layout tests as well and stop supporting the # incremental_results.json file format. if file.filename == "full_results.json" and test_type == "layout-tests": update_succeeded |= JsonResults.update(master, builder, test_type, file.value, is_full_results_format=True) if not update_succeeded: errors.append( "Upload failed, master: %s, builder: %s, test_type: %s, name: %s." % (master, builder, test_type, file.filename)) if errors: messages = "FAIL: " + "; ".join(errors) logging.warning(messages) self.response.set_status(500, messages) self.response.out.write("FAIL") else: self.response.set_status(200) self.response.out.write("OK")
def _get_file(cls, master, builder, test_type, filename): files = TestFile.get_files(master, builder, test_type, filename) if files: return files[0] file = TestFile() file.master = master file.builder = builder file.test_type = test_type file.name = filename file.data = "" return file
def post(self): file_params = self.request.POST.getall(PARAM_FILE) if not file_params: self.response.out.write("FAIL: missing upload file field.") return builder = self.request.get(PARAM_BUILDER) if not builder: self.response.out.write("FAIL: missing builder parameter.") return master = self.request.get(PARAM_MASTER) test_type = self.request.get(PARAM_TEST_TYPE) incremental = self.request.get(PARAM_INCREMENTAL) logging.debug( "Processing upload request, master: %s, builder: %s, test_type: %s.", master, builder, test_type) # There are two possible types of each file_params in the request: # one file item or a list of file items. # Normalize file_params to a file item list. files = [] logging.debug("test: %s, type:%s", file_params, type(file_params)) for item in file_params: if not isinstance(item, list) and not isinstance(item, tuple): item = [item] files.extend(item) errors = [] for file in files: filename = file.filename.lower() if ((incremental and filename == "results.json") or (filename == "incremental_results.json")): # Merge incremental json results. update_succeeded = JsonResults.update(master, builder, test_type, file.value) else: update_succeeded = TestFile.add_file(master, builder, test_type, file.filename, file.value) if not update_succeeded: errors.append( "Upload failed, master: %s, builder: %s, test_type: %s, name: %s." % (master, builder, test_type, file.filename)) if errors: messages = "FAIL: " + "; ".join(errors) logging.warning(messages) self.response.set_status(500, messages) self.response.out.write("FAIL") else: self.response.set_status(200) self.response.out.write("OK")
def post(self): file_params = self.request.POST.getall(PARAM_FILE) if not file_params: self.response.out.write("FAIL: missing upload file field.") return builder = self.request.get(PARAM_BUILDER) if not builder: self.response.out.write("FAIL: missing builder parameter.") return master = self.request.get(PARAM_MASTER) test_type = self.request.get(PARAM_TEST_TYPE) incremental = self.request.get(PARAM_INCREMENTAL) logging.debug( "Processing upload request, master: %s, builder: %s, test_type: %s.", master, builder, test_type) # There are two possible types of each file_params in the request: # one file item or a list of file items. # Normalize file_params to a file item list. files = [] logging.debug("test: %s, type:%s", file_params, type(file_params)) for item in file_params: if not isinstance(item, list) and not isinstance(item, tuple): item = [item] files.extend(item) errors = [] for file in files: filename = file.filename.lower() if ((incremental and filename == "results.json") or (filename == "incremental_results.json")): # Merge incremental json results. update_succeeded = JsonResults.update(master, builder, test_type, file.value) else: update_succeeded = TestFile.add_file( master, builder, test_type, file.filename, file.value) if not update_succeeded: errors.append( "Upload failed, master: %s, builder: %s, test_type: %s, name: %s." % (master, builder, test_type, file.filename)) if errors: messages = "FAIL: " + "; ".join(errors) logging.warning(messages) self.response.set_status(500, messages) self.response.out.write("FAIL") else: self.response.set_status(200) self.response.out.write("OK")
def update(cls, builder, test_type, incremental): """Update datastore json file data by merging it with incremental json file. Args: builder: builder name. test_type: type of test results. incremental: incremental json file data to merge. Returns: TestFile object if update succeeds or None on failure. """ files = TestFile.get_files(builder, test_type, JSON_RESULTS_FILE) if files: file = files[0] new_results = cls.merge(builder, file.data, incremental) else: # Use the incremental data if there is no aggregated file to merge. file = TestFile() file.builder = builder file.test_type = test_type file.name = JSON_RESULTS_FILE new_results = incremental logging.info("No existing json results, incremental json is saved.") if not new_results: return None if not file.save(new_results): return None return file
def _get_file_content(self, builder, test_type, name): """Return content of the file that matches builder and file name. Args: builder: builder name test_type: type of the test name: file name """ files = TestFile.get_files( builder, test_type, name, load_data=True, limit=1) if not files: logging.info("File not found, builder: %s, test_type: %s, name: %s.", builder, test_type, name) return None return files[0].data
def update_file(cls, master, builder, test_type, incremental, is_full_results_format, filename, num_runs): files = TestFile.get_files(master, builder, test_type, filename) if files: file = files[0] else: file = TestFile() file.master = master file.builder = builder file.test_type = test_type file.name = filename file.data = "" new_results = cls.merge(builder, file.data, incremental, is_full_results_format, num_runs) if not new_results or not file.save(new_results): logging.info("Update failed, master: %s, builder: %s, test_type: %s, name: %s." % (master, builder, test_type, filename)) return False return True
def _get_file_content(self, master, builder, test_type, build_number, name): """Return content of the file that matches builder and file name. Args: builder: builder name test_type: type of the test build_number: build number, or 'latest' name: file name """ files = TestFile.get_files( master, builder, test_type, build_number, name, load_data=True, limit=1) if not files: logging.info("File not found, master %s, builder: %s, test_type: %s, build_number: %s, name: %s.", master, builder, test_type, build_number, name) return None, None return files[0].data, files[0].date
def get(self): key = self.request.get(PARAM_KEY) master = self.request.get(PARAM_MASTER) builder = self.request.get(PARAM_BUILDER) test_type = self.request.get(PARAM_TEST_TYPE) name = self.request.get(PARAM_NAME) num_files = self.request.get(PARAM_NUM_FILES) before = self.request.get(PARAM_BEFORE) logging.debug( "Deleting File, master: %s, builder: %s, test_type: %s, name: %s, before: %s, key: %s.", master, builder, test_type, name, before, key) limit = int(num_files) if num_files else 1 num_deleted = TestFile.delete_file(key, master, builder, test_type, name, before, limit) self.response.set_status(200) self.response.out.write("Deleted %d files." % num_deleted)
def _get_file_content(self, builder, test_type, name): """Return content of the file that matches builder and file name. Args: builder: builder name test_type: type of the test name: file name """ files = TestFile.get_files(builder, test_type, name, 1) if not files: logging.info( "File not found, builder: %s, test_type: %s, name: %s.", builder, test_type, name) return blob_key = files[0].blob_key blob_info = blobstore.get(blob_key) if blob_info: self.send_blob(blob_info, "text/plain")
def get(self): key = self.request.get(PARAM_KEY) # Intentionally don't munge the master from deprecated names here. # Assume anyone deleting files wants explicit control. master = self.request.get(PARAM_MASTER) builder = self.request.get(PARAM_BUILDER) test_type = self.request.get(PARAM_TEST_TYPE) build_number = self.request.get(PARAM_BUILD_NUMBER, default_value=None) name = self.request.get(PARAM_NAME) num_files = self.request.get(PARAM_NUM_FILES) before = self.request.get(PARAM_BEFORE) logging.debug( "Deleting File, master: %s, builder: %s, test_type: %s, build_number: %s, name: %s, before: %s, key: %s.", master, builder, test_type, build_number, name, before, key) limit = int(num_files) if num_files else 1 num_deleted = TestFile.delete_file(key, master, builder, test_type, build_number, name, before, limit) self.response.set_status(200) self.response.out.write("Deleted %d files." % num_deleted)
def _get_file_content(self, builder, test_type, name): """Return content of the file that matches builder and file name. Args: builder: builder name test_type: type of the test name: file name """ files = TestFile.get_files(builder, test_type, name, load_data=True, limit=1) if not files: logging.info( "File not found, builder: %s, test_type: %s, name: %s.", builder, test_type, name) return None return files[0].data
def update_file(cls, master, builder, test_type, incremental, filename, num_runs): files = TestFile.get_files(master, builder, test_type, filename) if files: file = files[0] new_results = cls.merge(builder, file.data, incremental, num_runs) else: # Use the incremental data if there is no aggregated file to merge. file = TestFile() file.master = master file.builder = builder file.test_type = test_type file.name = filename new_results = incremental logging.info("No existing json results, incremental json is saved.") if not new_results or not file.save(new_results): logging.info("Update failed, master: %s, builder: %s, test_type: %s, name: %s." % (master, builder, test_type, filename)) return False return True
def update_file(cls, master, builder, test_type, incremental, filename, num_runs): files = TestFile.get_files(master, builder, test_type, filename) if files: file = files[0] new_results = cls.merge(builder, file.data, incremental, num_runs) else: # Use the incremental data if there is no aggregated file to merge. file = TestFile() file.master = master file.builder = builder file.test_type = test_type file.name = filename new_results = incremental logging.info( "No existing json results, incremental json is saved.") if not new_results or not file.save(new_results): logging.info( "Update failed, master: %s, builder: %s, test_type: %s, name: %s." % (master, builder, test_type, filename)) return False return True
def post(self): file_params = self.request.POST.getall(PARAM_FILE) if not file_params: self.response.out.write("FAIL: missing upload file field.") return builder = self.request.get(PARAM_BUILDER) if not builder: self.response.out.write("FAIL: missing builder parameter.") return master_parameter = self.request.get(PARAM_MASTER) master_data = master_config.getMasterByMasterName(master_parameter) if master_data: deprecated_master = master_parameter master = master_data['url_name'] else: deprecated_master = None master = master_parameter test_type = self.request.get(PARAM_TEST_TYPE) logging.debug( "Processing upload request, master: %s, builder: %s, test_type: %s.", master, builder, test_type) # There are two possible types of each file_params in the request: # one file item or a list of file items. # Normalize file_params to a file item list. files = [] logging.debug("test: %s, type:%s", file_params, type(file_params)) for item in file_params: if not isinstance(item, list) and not isinstance(item, tuple): item = [item] files.extend(item) errors = [] final_status_code = 200 for file in files: file_json = JsonResults._load_json(file.value) if file.filename == "incremental_results.json": # FIXME: Ferret out and eliminate remaining incremental_results.json producers. logging.info( "incremental_results.json received from master: %s, builder: %s, test_type: %s.", master, builder, test_type) status_string, status_code = JsonResults.update( master, builder, test_type, file_json, deprecated_master=deprecated_master, is_full_results_format=False) else: try: build_number = int(file_json.get('build_number', 0)) status_string, status_code = TestFile.add_file( master, builder, test_type, build_number, file.filename, file.value) except (ValueError, TypeError): status_code = 403 status_string = 'Could not cast the build_number field in the json to an integer.' if status_code == 200: logging.info(status_string) else: logging.error(status_string) errors.append(status_string) final_status_code = status_code if status_code == 200 and file.filename == "full_results.json": status_string, status_code = JsonResults.update( master, builder, test_type, file_json, deprecated_master=deprecated_master, is_full_results_format=True) if status_code == 200: logging.info(status_string) else: logging.error(status_string) errors.append(status_string) final_status_code = status_code if errors: messages = "FAIL: " + "; ".join(errors) self.response.set_status(final_status_code, messages) self.response.out.write(messages) else: self.response.set_status(200) self.response.out.write("OK")
def update_file(cls, builder, file, incremental_json, num_runs): new_results, status_code = cls.merge(builder, file.data, incremental_json, num_runs) if status_code != 200: return new_results, status_code return TestFile.save_file(file, new_results)
def post(self): file_params = self.request.POST.getall(PARAM_FILE) if not file_params: self.response.out.write("FAIL: missing upload file field.") return builder = self.request.get(PARAM_BUILDER) if not builder: self.response.out.write("FAIL: missing builder parameter.") return master_parameter = self.request.get(PARAM_MASTER) master_data = master_config.getMasterByMasterName(master_parameter) if master_data: deprecated_master = master_parameter master = master_data['url_name'] else: deprecated_master = None master = master_parameter test_type = self.request.get(PARAM_TEST_TYPE) logging.debug( "Processing upload request, master: %s, builder: %s, test_type: %s.", master, builder, test_type) # There are two possible types of each file_params in the request: # one file item or a list of file items. # Normalize file_params to a file item list. files = [] logging.debug("test: %s, type:%s", file_params, type(file_params)) for item in file_params: if not isinstance(item, list) and not isinstance(item, tuple): item = [item] files.extend(item) errors = [] final_status_code = 200 for file in files: file_json = JsonResults._load_json(file.value) if file.filename == "incremental_results.json": # FIXME: Ferret out and eliminate remaining incremental_results.json producers. logging.info("incremental_results.json received from master: %s, builder: %s, test_type: %s.", master, builder, test_type) status_string, status_code = JsonResults.update(master, builder, test_type, file_json, deprecated_master=deprecated_master, is_full_results_format=False) else: try: build_number = int(file_json.get('build_number', 0)) status_string, status_code = TestFile.add_file(master, builder, test_type, build_number, file.filename, file.value) except (ValueError, TypeError): status_code = 403 status_string = 'Could not cast the build_number field in the json to an integer.' if status_code == 200: logging.info(status_string) else: logging.error(status_string) errors.append(status_string) final_status_code = status_code if status_code == 200 and file.filename == "full_results.json": status_string, status_code = JsonResults.update(master, builder, test_type, file_json, deprecated_master=deprecated_master, is_full_results_format=True) if status_code == 200: logging.info(status_string) else: logging.error(status_string) errors.append(status_string) final_status_code = status_code if errors: messages = "FAIL: " + "; ".join(errors) self.response.set_status(final_status_code, messages) self.response.out.write(messages) else: self.response.set_status(200) self.response.out.write("OK")