def get_workspace(self, environ, start_response, job_id, build_id): """ Get workspace archive """ if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isfile(self._build_workspace_file(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) try: ifh = open(self._build_workspace_file(job_id, build_id)) except IOError: return response.send_error(start_response, 500, constants.ERROR_BUILD_READ_FAILED) file_len = os.path.getsize(self._build_workspace_file( job_id, build_id)) return response.send_response_file(environ, start_response, 200, ifh, file_len)
def handle_request(self, environ, start_response, method, job_id, build_id, parts): """ Handle requests related to build artifacts """ if validators.validate_job_id(job_id) == None: self.log.error('Invalid job_id: %r' % job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if len(parts) == 0: if method == 'POST': return self.create_or_update_artifact(environ, start_response, job_id, build_id) else: return response.send_error(start_response, 400) elif len(parts) == 1: if method == 'GET': return self.get_artifact(environ, start_response, job_id, build_id, parts[0]) elif method == 'PUT': return self.create_or_update_artifact(environ, start_response, job_id, build_id, parts[0]) elif method == 'DELETE': return self.delete_artifact(start_response, job_id, build_id, parts[0]) else: return response.send_error(start_response, 400) elif len(parts) == 2: if method == 'GET': return self.get_artifact(environ, start_response, job_id, build_id, parts[0]) else: return response.send_error(start_response, 400) return response.send_response(start_response, 400)
def update_build_state(self, request, job_id, build_id): """ Update build state """ try: build_state = json.load(request.body_file) except ValueError: self.log.exception('Failed to load build state') return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_PAYLOAD) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if self.cephmonitors: storage_backend = storage.CephFSStorage(','.join(self.cephmonitors)) else: storage_backend = storage.LocalFSStorage() with storage_backend as store: if not store.isdir(self._build_dir(job_id, build_id)): return webob.Response(status=404, body=constants.ERROR_BUILD_NOT_FOUND) try: with store.open(self._build_state_file(job_id, build_id), 'wb') as fileo: json.dump(build_state, fileo) except: self.log.exception('Failed to write build state, job_id %s, build %s', job_id, build_id) return webob.Response(status=500, body=constants.ERROR_BUILD_WRITE_FAILED) return webob.Response(status=200, body=json.dumps({'job_id': job_id, 'build_number': int(build_id), 'state': build_state}), content_type="application/json")
def handle_request(self, request, job_id, build_id, parts): """ Handle requests related to build artifacts """ if validators.validate_job_id(job_id) == None: self.log.error('Invalid job_id: %r' % job_id) return webob.Response(status=400, body=constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if len(parts) == 0: if request.method == 'POST': return self.create_or_update_artifact(request, job_id, build_id) else: if validators.validate_artifact_id(parts[0]) != parts[0]: return webob.Response(status=400, body=constants.ERROR_ARTIFACT_INVALID_ID) if len(parts) == 1: if request.method == 'GET': return self.get_artifact(job_id, build_id, parts[0]) elif request.method == 'PUT': return self.create_or_update_artifact(request, job_id, build_id, parts[0]) elif request.method == 'DELETE': return self.delete_artifact(job_id, build_id, parts[0]) elif len(parts) == 2: if request.method == 'GET': return self.get_artifact(job_id, build_id, parts[0]) return webob.Response(status=400)
def update_workspace(self, environ, start_response, job_id, build_id): """ Store workspace archive """ if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) ifh, data_len = request.get_request_data_handle_and_length(environ) try: ofh = open(self._build_workspace_file(job_id, build_id), 'wb') except IOError: return response.send_error(start_response, 500, constants.ERROR_BUILD_WRITE_FAILED) try: while data_len > 0: read_len = data_len if read_len > 1024*128: read_len = 1024*128 data = ifh.read(read_len) ofh.write(data) data_len = data_len - len(data) except IOError: ofh.close() return response.send_error(start_response, 500, constants.ERROR_BUILD_WRITE_FAILED) ofh.close() return response.send_response(start_response, 204)
def get_build_state(self, job_id, build_id): """ Get job state """ if validators.validate_build_id(build_id) != build_id: return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if self.cephmonitors: storage_backend = storage.CephFSStorage(','.join(self.cephmonitors)) else: storage_backend = storage.LocalFSStorage() build_data = None with storage_backend as store: if not store.isdir(self._build_dir(job_id, build_id)): return webob.Response(status=404, body=constants.ERROR_JOB_NOT_FOUND) for _ in range(10): try: with store.open(self._build_state_file(job_id, build_id), 'rb') as fileo: build_state = json.load(fileo) build_data = json.dumps({'job_id': job_id, 'build_number': build_id, 'state': build_state}) break except (storage.NotFound, ValueError): time.sleep(0.1) except: self.log.exception("Exception while reading build state") return webob.Response(status=500, body=constants.ERROR_INTERNAL) if not build_data: return webob.Response(status=409, body=constants.ERROR_BUILD_LOCKED) return webob.Response(status=200, body=build_data, content_type="application/json")
def get_build_state(self, start_response, job_id, build_id): """ Get job state """ if validators.validate_job_id(job_id) != job_id: return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_JOB_NOT_FOUND) build_data = None for _ in range(10): try: build_state = json.load( file(self._build_state_file(job_id, build_id), 'rb')) build_data = json.dumps({ 'job_id': job_id, 'build_number': build_id, 'state': build_state }) break except (OSError, ValueError): time.sleep(0.1) if not build_data: return response.send_error(start_response, 409, constants.ERROR_BUILD_LOCKED) return response.send_response(start_response, 200, build_data)
def update_console_log(self, environ, start_response, job_id, build_id): """ Append content to the console log """ if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) try: file(self._console_log_file(job_id, build_id), 'ab').write(request.read_request_data(environ)) except IOError: # can be ignored, we just return empty log pass return response.send_response(start_response, 204)
def delete_build(self, start_response, job_id, build_id): """ Delete a specific build and all related data """ if validators.validate_job_id(job_id) == None: self.log.debug('Invalid job_id: %r' % job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) self.log.debug("delete build %s/%s" % (job_id, build_id)) try: shutil.rmtree(self._build_dir(job_id, build_id)) except OSError: return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) return response.send_response(start_response, 204)
def get_console_log(self, start_response, job_id, build_id): """ Return contents of the console log """ if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) console_log = '' try: console_log = file(self._console_log_file(job_id, build_id), 'rb').read() except IOError: # can be ignored, we just return empty log pass return response.send_response(start_response, 200, console_log, content_type="text/plain")
def delete_workspace(self, start_response, job_id, build_id): """ Delete workspace archive """ if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isfile(self._build_workspace_file(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) try: os.unlink(self._build_workspace_file(job_id, build_id)) except IOError: return response.send_error(start_response, 500, constants.ERROR_BUILD_WRITE_FAILED) return response.send_response(start_response, 204)
def get_workspace(self, environ, start_response, job_id, build_id): """ Get workspace archive """ if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isfile(self._build_workspace_file(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) try: ifh = open(self._build_workspace_file(job_id, build_id)) except IOError: return response.send_error(start_response, 500, constants.ERROR_BUILD_READ_FAILED) file_len = os.path.getsize(self._build_workspace_file(job_id, build_id)) return response.send_response_file(environ, start_response, 200, ifh, file_len)
def get_build_state(self, start_response, job_id, build_id): """ Get job state """ if validators.validate_job_id(job_id) != job_id: return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_JOB_NOT_FOUND) build_data = None for _ in range(10): try: build_state = json.load(file(self._build_state_file(job_id, build_id), 'rb')) build_data = json.dumps({'job_id': job_id, 'build_number': build_id, 'state': build_state}) break except (OSError, ValueError): time.sleep(0.1) if not build_data: return response.send_error(start_response, 409, constants.ERROR_BUILD_LOCKED) return response.send_response(start_response, 200, build_data)
def delete_build(self, job_id, build_id): """ Delete a specific build and all related data """ if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if self.cephmonitors: storage_backend = storage.CephFSStorage(','.join(self.cephmonitors)) else: storage_backend = storage.LocalFSStorage() with storage_backend as store: if not store.isdir(self._build_dir(job_id, build_id)): return webob.Response(status=404, body=constants.ERROR_BUILD_NOT_FOUND) try: store.rmtree(self._build_dir(job_id, build_id)) except storage.NotFound: return webob.Response(status=204) except: self.log.exception("Exception on delete build") return webob.Response(status=404, body=constants.ERROR_BUILD_NOT_FOUND) return webob.Response(status=204)
def update_console_log(self, request, job_id, build_id): """ Append content to the console log """ if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if self.cephmonitors: storage_backend = storage.CephFSStorage(','.join(self.cephmonitors)) else: storage_backend = storage.LocalFSStorage() with storage_backend as store: if not store.isdir(self._build_dir(job_id, build_id)): return webob.Response(status=404, body=constants.ERROR_BUILD_NOT_FOUND) try: log = request.body_file.read() with store.open(self._console_log_file(job_id, build_id), 'ab') as fileo: fileo.write(log) except: self.log.exception("Exception while updating console log") # FIXME: ignored for now return webob.Response(status=204)
def delete_workspace(self, job_id, build_id): """ Delete workspace archive """ if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if self.cephmonitors: storage_backend = storage.CephFSStorage(','.join(self.cephmonitors)) else: storage_backend = storage.LocalFSStorage() with storage_backend as store: if not store.isfile(self._build_workspace_file(job_id, build_id)): return webob.Response(status=404, body=constants.ERROR_BUILD_NOT_FOUND) try: store.unlink(self._build_workspace_file(job_id, build_id)) except storage.NotFound: return webob.Response(status=500, body=constants.ERROR_BUILD_WRITE_FAILED) except: self.log.exception("Exception while deleting workspace") return webob.Response(status=500, body=constants.ERROR_BUILD_WRITE_FAILED) return webob.Response(status=204)
def get_console_log(self, job_id, build_id): """ Return contents of the console log """ if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if self.cephmonitors: storage_backend = storage.CephFSStorage(','.join(self.cephmonitors)) else: storage_backend = storage.LocalFSStorage() console_log = '' with storage_backend as store: if not store.isdir(self._build_dir(job_id, build_id)): return webob.Response(status=404, body=constants.ERROR_BUILD_NOT_FOUND) try: with store.open(self._console_log_file(job_id, build_id), 'rb') as fileo: console_log = fileo.read() except storage.NotFound: # can be ignored, we just return empty log pass except: self.log.exception("Exception while reading console log") return webob.Response(status=200, body=console_log, content_type="text/plain")
def update_workspace(self, environ, start_response, job_id, build_id): """ Store workspace archive """ if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) ifh, data_len = request.get_request_data_handle_and_length(environ) try: ofh = open(self._build_workspace_file(job_id, build_id), 'wb') except IOError: return response.send_error(start_response, 500, constants.ERROR_BUILD_WRITE_FAILED) try: while data_len > 0: read_len = data_len if read_len > 1024 * 128: read_len = 1024 * 128 data = ifh.read(read_len) ofh.write(data) data_len = data_len - len(data) except IOError: ofh.close() return response.send_error(start_response, 500, constants.ERROR_BUILD_WRITE_FAILED) ofh.close() return response.send_response(start_response, 204)
def update_workspace(self, request, job_id, build_id): """ Store workspace archive """ if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if self.cephmonitors: storage_backend = storage.CephFSStorage(','.join(self.cephmonitors)) else: storage_backend = storage.LocalFSStorage() with storage_backend as store: if not store.isdir(self._build_dir(job_id, build_id)): return webob.Response(status=404, body=constants.ERROR_BUILD_NOT_FOUND) data_len = request.content_length ifh = request.body_file try: ofh = store.open(self._build_workspace_file(job_id, build_id), 'wb') except: self.log.exception("Failed to open workspace for writing") return webob.Response(status=500, body=constants.ERROR_BUILD_WRITE_FAILED) try: while data_len > 0: read_len = data_len if read_len > 1024*128: read_len = 1024*128 data = ifh.read(read_len) ofh.write(data) data_len = data_len - len(data) except: self.log.exception("Exception while updating workspace") ofh.close() return webob.Response(status=500, body=constants.ERROR_BUILD_WRITE_FAILED) ofh.close() return webob.Response(status=204)
def update_build_state(self, environ, start_response, job_id, build_id): """ Update build state """ try: build_state = json.loads(request.read_request_data(environ)) except ValueError: self.log.error('Failed to load build state') return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_PAYLOAD) if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) try: file(self._build_state_file(job_id, build_id), 'wb').write(json.dumps(build_state)) except IOError: self.log.debug('Failed to write build state, job_id %s, build %s', job_id, build_id) return response.send_error(start_response, 500, constants.ERROR_BUILD_WRITE_FAILED) return response.send_response(start_response, 200, json.dumps({'job_id': job_id, 'build_number': int(build_id), 'state': build_state}))
def update_build_state(self, environ, start_response, job_id, build_id): """ Update build state """ try: build_state = json.loads(request.read_request_data(environ)) except ValueError: self.log.error('Failed to load build state') return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_PAYLOAD) if validators.validate_job_id(job_id) == None: self.log.error("Job_id validation failure, '%s'", job_id) return response.send_error(start_response, 400, constants.ERROR_JOB_INVALID_ID) if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return response.send_error(start_response, 400, constants.ERROR_BUILD_INVALID_ID) if not os.path.isdir(self._build_dir(job_id, build_id)): return response.send_error(start_response, 404, constants.ERROR_BUILD_NOT_FOUND) try: file(self._build_state_file(job_id, build_id), 'wb').write(json.dumps(build_state)) except IOError: self.log.debug('Failed to write build state, job_id %s, build %s', job_id, build_id) return response.send_error(start_response, 500, constants.ERROR_BUILD_WRITE_FAILED) return response.send_response( start_response, 200, json.dumps({ 'job_id': job_id, 'build_number': int(build_id), 'state': build_state }))
def get_workspace(self, job_id, build_id): """ Get workspace archive """ if validators.validate_build_id(build_id) != build_id: self.log.error("Build_id validation failure, '%s'", build_id) return webob.Response(status=400, body=constants.ERROR_BUILD_INVALID_ID) if self.cephmonitors: storage_backend = storage.CephFSStorage(','.join(self.cephmonitors)) else: storage_backend = storage.LocalFSStorage() with storage_backend as store: if not store.isfile(self._build_workspace_file(job_id, build_id)): return webob.Response(status=404, body=constants.ERROR_BUILD_NOT_FOUND) try: ifh = store.open(self._build_workspace_file(job_id, build_id), 'rb') except storage.NotFound: return webob.Response(status=500, body=constants.ERROR_BUILD_READ_FAILED) except: self.log.exception("Exception in get workspace") return webob.Response(status=500, body=constants.ERROR_BUILD_READ_FAILED) file_len = store.getsize(self._build_workspace_file(job_id, build_id)) return webob.Response(status=200, body_file=ifh, content_length=file_len, content_type="application/octet-stream")
def test_05_valid_build_ids(self): for build_id in self.valid_build_ids: assert validators.validate_build_id(build_id) == build_id, "Validator declined valid build_id %s" % build_id
def test_06_invalid_build_ids(self): for build_id in self.invalid_build_ids: assert validators.validate_build_id(build_id) is None, "Validator accepted invalid build_id %s" % build_id