def _on_request_body(self, data): try: self._request.body = data content_type = self._request.headers.get("Content-Type", "") if self._request.method in ("POST", "PATCH", "PUT"): if content_type.startswith( "application/x-www-form-urlencoded" ) and self.content_length < GLSetting.www_form_urlencoded_maximum_size: arguments = parse_qs_bytes(native_str(self._request.body)) for name, values in arguments.iteritems(): values = [v for v in values if v] if values: self._request.arguments.setdefault( name, []).extend(values) elif content_type.startswith( "application/x-www-form-urlencoded"): raise errors.InvalidInputFormat( "content type application/x-www-form-urlencoded not supported" ) elif content_type.startswith("multipart/form-data"): raise errors.InvalidInputFormat( "content type multipart/form-data not supported") self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def _on_request_body(self, data): try: self._request.body = data content_type = self._request.headers.get("Content-Type", "") if self._request.method in ("POST", "PATCH", "PUT"): if content_type.startswith("application/x-www-form-urlencoded"): raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported") elif content_type.startswith("multipart/form-data"): raise errors.InvalidInputFormat("content type multipart/form-data not supported") self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def db_update_submission(store, submission_id, request, finalize, language): context = store.find(Context, Context.id == unicode(request['context_id'])).one() if not context: log.err("Context requested: [%s] not found!" % request['context_id']) raise errors.ContextIdNotFound submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one() if not submission: log.err("Invalid Submission requested %s in PUT" % submission_id) raise errors.SubmissionIdNotFound # this may happen if a submission try to update a context if submission.context_id != context.id: log.err("Can't be changed context in a submission update") raise errors.ContextIdNotFound() if submission.mark != u'submission': log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark)) raise errors.SubmissionConcluded try: import_files(store, submission, request['files']) except Exception as excep: log.err("Submission update: files import fail: %s" % excep) log.exception(excep) raise excep try: wb_steps = request['wb_steps'] if finalize: steps = db_get_context_steps(store, context.id, language) verify_steps(steps, wb_steps) submission.wb_steps = wb_steps except Exception as excep: log.err("Submission update: fields validation fail: %s" % excep) log.exception(excep) raise excep try: import_receivers(store, submission, request['receivers'], required=finalize) except Exception as excep: log.err("Submission update: receiver import fail: %s" % excep) log.exception(excep) raise excep if finalize: submission.mark = u'finalize' # Finalized else: submission.mark = u'submission' # Submission submission_dict = wb_serialize_internaltip(submission) return submission_dict
def _on_request_body(self, data): try: self._request.body = data content_type = self._request.headers.get("Content-Type", "") if self._request.method in ("POST", "PATCH", "PUT"): if content_type.startswith("application/x-www-form-urlencoded") and self.content_length < GLSetting.www_form_urlencoded_maximum_size: arguments = parse_qs_bytes(native_str(self._request.body)) for name, values in arguments.iteritems(): values = [v for v in values if v] if values: self._request.arguments.setdefault(name, []).extend(values) elif content_type.startswith("application/x-www-form-urlencoded"): raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported") elif content_type.startswith("multipart/form-data"): raise errors.InvalidInputFormat("content type multipart/form-data not supported") self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language): context = store.find(Context, Context.id == unicode(request['context_id'])).one() if not context: log.err("Context requested: [%s] not found!" % request['context_id']) raise errors.ContextIdNotFound submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one() if not submission: log.err("Invalid Submission requested %s in PUT" % submission_id) raise errors.SubmissionIdNotFound # this may happen if a submission try to update a context if submission.context_id != context.id: log.err("Can't be changed context in a submission update") raise errors.ContextIdNotFound("Context are immutable") if submission.mark != u'submission': log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark)) raise errors.SubmissionConcluded try: import_files(store, submission, request['files'], finalize) except Exception as excep: log.err("Submission update: files import fail: %s" % excep) log.exception(excep) raise excep try: wb_steps = request['wb_steps'] if finalize: steps = db_get_context_steps(store, context.id, language) verify_steps(steps, wb_steps) submission.wb_steps = wb_steps except Exception as excep: log.err("Submission update: fields validation fail: %s" % excep) log.exception(excep) raise excep try: import_receivers(store, submission, request['receivers'], required=finalize) except Exception as excep: log.err("Submission update: receiver import fail: %s" % excep) log.exception(excep) raise excep if finalize: submission.mark = u'finalize' # Finalized else: submission.mark = u'submission' # Submission submission_dict = wb_serialize_internaltip(submission) return submission_dict
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language): context = store.find(Context, Context.id == unicode(request["context_id"])).one() if not context: log.err("Context requested: [%s] not found!" % request["context_id"]) raise errors.ContextIdNotFound submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one() if not submission: log.err("Invalid Submission requested %s in PUT" % submission_id) raise errors.SubmissionIdNotFound # this may happen if a submission try to update a context if submission.context_id != context.id: log.err("Can't be changed context in a submission update") raise errors.ContextIdNotFound("Context are immutable") if submission.mark != InternalTip._marker[0]: log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark)) raise errors.SubmissionConcluded try: import_files(store, submission, request["files"], finalize) except Exception as excep: log.err("Submission update: files import fail: %s" % excep) log.exception(excep) raise excep try: wb_fields = request["wb_fields"] fo = Fields(context.localized_fields, context.unique_fields) fo.validate_fields(wb_fields, language, strict_validation=finalize) submission.wb_fields = wb_fields except Exception as excep: log.err("Submission update: fields validation fail: %s" % excep) log.exception(excep) raise excep try: import_receivers(store, submission, request["receivers"], required=finalize) except Exception as excep: log.err("Submission update: receiver import fail: %s" % excep) log.exception(excep) raise excep if finalize: submission.mark = InternalTip._marker[1] # Finalized submission_dict = wb_serialize_internaltip(submission) return submission_dict
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language): context = store.find(Context, Context.id == unicode(request['context_id'])).one() if not context: log.err("Context requested: [%s] not found!" % request['context_id']) raise errors.ContextIdNotFound submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one() if not submission: log.err("Invalid Submission requested %s in PUT" % submission_id) raise errors.SubmissionIdNotFound # this may happen if a submission try to update a context if submission.context_id != context.id: log.err("Can't be changed context in a submission update") raise errors.ContextIdNotFound("Context are immutable") if submission.mark != InternalTip._marker[0]: log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark)) raise errors.SubmissionConcluded try: import_files(store, submission, request['files'], finalize) except Exception as excep: log.err("Submission update: files import fail: %s" % excep) log.exception(excep) raise excep try: wb_fields = request['wb_fields'] fo = Fields(context.localized_fields, context.unique_fields) fo.validate_fields(wb_fields, language, strict_validation=finalize) submission.wb_fields = wb_fields except Exception as excep: log.err("Submission update: fields validation fail: %s" % excep) log.exception(excep) raise excep try: import_receivers(store, submission, request['receivers'], required=finalize) except Exception as excep: log.err("Submission update: receiver import fail: %s" % excep) log.exception(excep) raise excep if finalize: submission.mark = InternalTip._marker[1] # Finalized submission_dict = wb_serialize_internaltip(submission) return submission_dict
def on_error(self, excep): log.err("Exception while running %s" % self.name) log.exception(excep) extract_exception_traceback_and_schedule_email(excep)
def on_error(self, excep): error = "Job %s died with runtime %.4f [low: %.4f, high: %.4f]" % \ (self.name, self.mean_time, self.low_time, self.high_time) log.err(error) log.exception(excep) extract_exception_traceback_and_schedule_email(excep)
def _on_headers(self, data): try: data = native_str(data.decode("latin1")) eol = data.find("\r\n") start_line = data[:eol] try: method, uri, version = start_line.split(" ") except ValueError: raise _BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise _BadRequestException( "Malformed HTTP version in HTTP Request-Line") headers = httputil.HTTPHeaders.parse(data[eol:]) self._request = HTTPRequest( connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip) try: self.content_length = int(headers.get("Content-Length", 0)) except ValueError: raise _BadRequestException("Malformed Content-Length header") # we always use secure temporary files in case of large json or file uploads if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None: self._contentbuffer = StringIO('') else: self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path) if headers.get("Expect") == "100-continue": self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n") c_d_header = self._request.headers.get("Content-Disposition") if c_d_header is not None: key, pdict = parse_header(c_d_header) if key != 'attachment' or 'filename' not in pdict: raise _BadRequestException("Malformed Content-Disposition header") self.file_upload = True self.uploaded_file['filename'] = pdict['filename'] self.uploaded_file['content_type'] = self._request.headers.get("Content-Type", 'application/octet-stream') self.uploaded_file['body'] = self._contentbuffer self.uploaded_file['body_len'] = int(self.content_length) self.uploaded_file['body_filepath'] = self._contentbuffer.filepath megabytes = int(self.content_length) / (1024 * 1024) if self.file_upload: limit_type = "upload" limit = GLSetting.memory_copy.maximum_filesize else: limit_type = "json" limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize # is 1MB probably too high. probably this variable must be in kB # less than 1 megabytes is always accepted if megabytes > limit: log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit)) # In HTTP Protocol errors need to be managed differently than handlers raise errors.HTTPRawLimitReach if self.content_length > 0: self.setRawMode() return elif self.file_upload: self._on_request_body(self.uploaded_file) self.file_upload = False self.uploaded_file = {} return self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()
def update_submission(store, id, request, finalize, language=GLSetting.memory_copy.default_language): context = store.find(Context, Context.id == unicode(request['context_gus'])).one() if not context: log.err("Context requested: [%s] not found!" % request['context_gus']) raise errors.ContextGusNotFound submission = store.find(InternalTip, InternalTip.id == unicode(id)).one() if not submission: log.debug("Creating a new submission in update!") submission = InternalTip() submission.escalation_threshold = context.escalation_threshold submission.access_limit = context.tip_max_access submission.download_limit = context.file_max_download submission.expiration_date = utc_future_date(seconds=context.tip_timetolive) submission.pertinence_counter = 0 submission.context_id = context.id submission.creation_date = datetime_now() submission.mark = InternalTip._marker[0] # Submission try: store.add(submission) except Exception as excep: log.err("Storm/SQL Error: %s (update_submission)" % excep) raise errors.InternalServerError("Unable to commit on DB") # this may happen if a submission try to update a context if submission.context_id != context.id: log.err("Can't be changed context in a submission update") raise errors.ContextGusNotFound("Context are immutable") if submission.mark != InternalTip._marker[0]: log.err("Submission %s do not permit update (status %s)" % (id, submission.mark)) raise errors.SubmissionConcluded files = request.get('files', []) try: import_files(store, submission, files, finalize) except Exception as excep: log.err("Submission update: files import fail: %s" % excep) log.exception(excep) raise excep wb_fields = request.get('wb_fields', []) try: fo = Fields(context.localized_fields, context.unique_fields) fo.validate_fields(wb_fields, strict_validation=finalize) submission.wb_fields = wb_fields except Exception as excep: log.err("Submission update: fields validation fail: %s" % excep) log.exception(excep) raise excep receivers = request.get('receivers', []) try: import_receivers(store, submission, receivers, required=finalize) except Exception as excep: log.err("Submission update: receiver import fail: %s" % excep) log.exception(excep) raise excep if finalize: submission.mark = InternalTip._marker[1] # Finalized submission_dict = wb_serialize_internaltip(submission) return submission_dict
def _on_headers(self, data): try: data = native_str(data.decode("latin1")) eol = data.find("\r\n") start_line = data[:eol] try: method, uri, version = start_line.split(" ") except ValueError: raise _BadRequestException("Malformed HTTP request line") if not version.startswith("HTTP/"): raise _BadRequestException("Malformed HTTP version in HTTP Request-Line") headers = httputil.HTTPHeaders.parse(data[eol:]) self._request = HTTPRequest( connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip ) content_length = int(headers.get("Content-Length", 0)) self.content_length = content_length if content_length: if headers.get("Expect") == "100-continue": self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n") if content_length < 100000: self._contentbuffer = StringIO("") else: self._contentbuffer = TemporaryFile() c_d_header = self._request.headers.get("Content-Disposition") if c_d_header is not None: c_d_header = c_d_header.lower() m = content_disposition_re.match(c_d_header) if m is None: raise Exception self.file_upload = True self.uploaded_file["filename"] = m.group(1) self.uploaded_file["content_type"] = self._request.headers.get( "Content-Type", "application/octet-stream" ) self.uploaded_file["body"] = self._contentbuffer self.uploaded_file["body_len"] = int(content_length) megabytes = int(content_length) / (1024 * 1024) if self.file_upload: limit_type = "upload" limit = GLSetting.memory_copy.maximum_filesize else: limit_type = "json" limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize # is 1MB probably too high. probably this variable must be # in kB # less than 1 megabytes is always accepted if megabytes > limit: log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit)) # In HTTP Protocol errors need to be managed differently than handlers raise errors.HTTPRawLimitReach self.setRawMode() return self.request_callback(self._request) except Exception as exception: log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception)) log.exception(exception) if self._request: self._request.finish() if self.transport: self.transport.loseConnection()