Exemple #1
0
 def _on_request_body(self, data):
     try:
         self._request.body = data
         content_type = self._request.headers.get("Content-Type", "")
         if self._request.method in ("POST", "PATCH", "PUT"):
             if content_type.startswith(
                     "application/x-www-form-urlencoded"
             ) and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
                 arguments = parse_qs_bytes(native_str(self._request.body))
                 for name, values in arguments.iteritems():
                     values = [v for v in values if v]
                     if values:
                         self._request.arguments.setdefault(
                             name, []).extend(values)
             elif content_type.startswith(
                     "application/x-www-form-urlencoded"):
                 raise errors.InvalidInputFormat(
                     "content type application/x-www-form-urlencoded not supported"
                 )
             elif content_type.startswith("multipart/form-data"):
                 raise errors.InvalidInputFormat(
                     "content type multipart/form-data not supported")
         self.request_callback(self._request)
     except Exception as exception:
         log.msg("Malformed HTTP request from %s: %s" %
                 (self._remote_ip, exception))
         log.exception(exception)
         if self._request:
             self._request.finish()
         if self.transport:
             self.transport.loseConnection()
Exemple #2
0
 def _on_request_body(self, data):
     try:
         self._request.body = data
         content_type = self._request.headers.get("Content-Type", "")
         if self._request.method in ("POST", "PATCH", "PUT"):
             if content_type.startswith("application/x-www-form-urlencoded"):
                 raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
             elif content_type.startswith("multipart/form-data"):
                 raise errors.InvalidInputFormat("content type multipart/form-data not supported")
         self.request_callback(self._request)
     except Exception as exception:
         log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
         log.exception(exception)
         if self._request:
             self._request.finish()
         if self.transport:
             self.transport.loseConnection()
Exemple #3
0
def db_update_submission(store, submission_id, request, finalize, language):
    context = store.find(Context,
                         Context.id == unicode(request['context_id'])).one()
    if not context:
        log.err("Context requested: [%s] not found!" % request['context_id'])
        raise errors.ContextIdNotFound

    submission = store.find(InternalTip,
                            InternalTip.id == unicode(submission_id)).one()
    if not submission:
        log.err("Invalid Submission requested %s in PUT" % submission_id)
        raise errors.SubmissionIdNotFound

    # this may happen if a submission try to update a context
    if submission.context_id != context.id:
        log.err("Can't be changed context in a submission update")
        raise errors.ContextIdNotFound()

    if submission.mark != u'submission':
        log.err("Submission %s do not permit update (status %s)" %
                (submission_id, submission.mark))
        raise errors.SubmissionConcluded

    try:
        import_files(store, submission, request['files'])
    except Exception as excep:
        log.err("Submission update: files import fail: %s" % excep)
        log.exception(excep)
        raise excep

    try:
        wb_steps = request['wb_steps']
        if finalize:
            steps = db_get_context_steps(store, context.id, language)
            verify_steps(steps, wb_steps)

        submission.wb_steps = wb_steps
    except Exception as excep:
        log.err("Submission update: fields validation fail: %s" % excep)
        log.exception(excep)
        raise excep

    try:
        import_receivers(store,
                         submission,
                         request['receivers'],
                         required=finalize)
    except Exception as excep:
        log.err("Submission update: receiver import fail: %s" % excep)
        log.exception(excep)
        raise excep

    if finalize:
        submission.mark = u'finalize'  # Finalized
    else:
        submission.mark = u'submission'  # Submission

    submission_dict = wb_serialize_internaltip(submission)
    return submission_dict
Exemple #4
0
 def _on_request_body(self, data):
     try:
         self._request.body = data
         content_type = self._request.headers.get("Content-Type", "")
         if self._request.method in ("POST", "PATCH", "PUT"):
             if content_type.startswith("application/x-www-form-urlencoded") and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
                 arguments = parse_qs_bytes(native_str(self._request.body))
                 for name, values in arguments.iteritems():
                     values = [v for v in values if v]
                     if values:
                         self._request.arguments.setdefault(name,
                                                            []).extend(values)
             elif content_type.startswith("application/x-www-form-urlencoded"):
                 raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
             elif content_type.startswith("multipart/form-data"):
                 raise errors.InvalidInputFormat("content type multipart/form-data not supported")
         self.request_callback(self._request)
     except Exception as exception:
         log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
         log.exception(exception)
         if self._request:
             self._request.finish()
         if self.transport:
             self.transport.loseConnection()
Exemple #5
0
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language):

    context = store.find(Context, Context.id == unicode(request['context_id'])).one()
    if not context:
        log.err("Context requested: [%s] not found!" % request['context_id'])
        raise errors.ContextIdNotFound

    submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one()
    if not submission:
        log.err("Invalid Submission requested %s in PUT" % submission_id)
        raise errors.SubmissionIdNotFound

    # this may happen if a submission try to update a context
    if submission.context_id != context.id:
        log.err("Can't be changed context in a submission update")
        raise errors.ContextIdNotFound("Context are immutable")

    if submission.mark != u'submission':
        log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark))
        raise errors.SubmissionConcluded

    try:
        import_files(store, submission, request['files'], finalize)
    except Exception as excep:
        log.err("Submission update: files import fail: %s" % excep)
        log.exception(excep)
        raise excep

    try:
        wb_steps = request['wb_steps']
        if finalize:
            steps = db_get_context_steps(store, context.id, language)
            verify_steps(steps, wb_steps)

        submission.wb_steps = wb_steps
    except Exception as excep:
        log.err("Submission update: fields validation fail: %s" % excep)
        log.exception(excep)
        raise excep

    try:
        import_receivers(store, submission, request['receivers'], required=finalize)
    except Exception as excep:
        log.err("Submission update: receiver import fail: %s" % excep)
        log.exception(excep)
        raise excep

    if finalize:
        submission.mark = u'finalize'  # Finalized
    else:
        submission.mark = u'submission' # Submission

    submission_dict = wb_serialize_internaltip(submission)
    return submission_dict
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language):

    context = store.find(Context, Context.id == unicode(request["context_id"])).one()
    if not context:
        log.err("Context requested: [%s] not found!" % request["context_id"])
        raise errors.ContextIdNotFound

    submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one()
    if not submission:
        log.err("Invalid Submission requested %s in PUT" % submission_id)
        raise errors.SubmissionIdNotFound

    # this may happen if a submission try to update a context
    if submission.context_id != context.id:
        log.err("Can't be changed context in a submission update")
        raise errors.ContextIdNotFound("Context are immutable")

    if submission.mark != InternalTip._marker[0]:
        log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark))
        raise errors.SubmissionConcluded

    try:
        import_files(store, submission, request["files"], finalize)
    except Exception as excep:
        log.err("Submission update: files import fail: %s" % excep)
        log.exception(excep)
        raise excep

    try:
        wb_fields = request["wb_fields"]
        fo = Fields(context.localized_fields, context.unique_fields)
        fo.validate_fields(wb_fields, language, strict_validation=finalize)
        submission.wb_fields = wb_fields
    except Exception as excep:
        log.err("Submission update: fields validation fail: %s" % excep)
        log.exception(excep)
        raise excep

    try:
        import_receivers(store, submission, request["receivers"], required=finalize)
    except Exception as excep:
        log.err("Submission update: receiver import fail: %s" % excep)
        log.exception(excep)
        raise excep

    if finalize:
        submission.mark = InternalTip._marker[1]  # Finalized

    submission_dict = wb_serialize_internaltip(submission)
    return submission_dict
Exemple #7
0
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language):

    context = store.find(Context, Context.id == unicode(request['context_id'])).one()
    if not context:
        log.err("Context requested: [%s] not found!" % request['context_id'])
        raise errors.ContextIdNotFound

    submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one()
    if not submission:
        log.err("Invalid Submission requested %s in PUT" % submission_id)
        raise errors.SubmissionIdNotFound

    # this may happen if a submission try to update a context
    if submission.context_id != context.id:
        log.err("Can't be changed context in a submission update")
        raise errors.ContextIdNotFound("Context are immutable")

    if submission.mark != InternalTip._marker[0]:
        log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark))
        raise errors.SubmissionConcluded

    try:
        import_files(store, submission, request['files'], finalize)
    except Exception as excep:
        log.err("Submission update: files import fail: %s" % excep)
        log.exception(excep)
        raise excep

    try:
        wb_fields = request['wb_fields']
        fo = Fields(context.localized_fields, context.unique_fields)
        fo.validate_fields(wb_fields, language, strict_validation=finalize)
        submission.wb_fields = wb_fields
    except Exception as excep:
        log.err("Submission update: fields validation fail: %s" % excep)
        log.exception(excep)
        raise excep

    try:
        import_receivers(store, submission, request['receivers'], required=finalize)
    except Exception as excep:
        log.err("Submission update: receiver import fail: %s" % excep)
        log.exception(excep)
        raise excep

    if finalize:
        submission.mark = InternalTip._marker[1] # Finalized

    submission_dict = wb_serialize_internaltip(submission)
    return submission_dict
Exemple #8
0
 def on_error(self, excep):
     log.err("Exception while running %s" % self.name)
     log.exception(excep)
     extract_exception_traceback_and_schedule_email(excep)
Exemple #9
0
 def on_error(self, excep):
     error = "Job %s died with runtime %.4f [low: %.4f, high: %.4f]" % \
             (self.name, self.mean_time, self.low_time, self.high_time)
     log.err(error)
     log.exception(excep)
     extract_exception_traceback_and_schedule_email(excep)
Exemple #10
0
    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException(
                    "Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])
            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=self._remote_ip)

            try:
                self.content_length = int(headers.get("Content-Length", 0))
            except ValueError:
                raise _BadRequestException("Malformed Content-Length header")

            # we always use secure temporary files in case of large json or file uploads
            if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
                self._contentbuffer = StringIO('')
            else:
                self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)

            if headers.get("Expect") == "100-continue":
                self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")

            c_d_header = self._request.headers.get("Content-Disposition")
            if c_d_header is not None:
                key, pdict = parse_header(c_d_header)
                if key != 'attachment' or 'filename' not in pdict:
                    raise _BadRequestException("Malformed Content-Disposition header")

                self.file_upload = True
                self.uploaded_file['filename'] = pdict['filename']
                self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
                                                                               'application/octet-stream')

                self.uploaded_file['body'] = self._contentbuffer
                self.uploaded_file['body_len'] = int(self.content_length)
                self.uploaded_file['body_filepath'] = self._contentbuffer.filepath

            megabytes = int(self.content_length) / (1024 * 1024)

            if self.file_upload:
                limit_type = "upload"
                limit = GLSetting.memory_copy.maximum_filesize
            else:
                limit_type = "json"
                limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
                # is 1MB probably too high. probably this variable must be in kB

            # less than 1 megabytes is always accepted
            if megabytes > limit:
                log.err("Tried %s request larger than expected (%dMb > %dMb)" %
                        (limit_type,
                         megabytes,
                         limit))

                # In HTTP Protocol errors need to be managed differently than handlers
                raise errors.HTTPRawLimitReach

            if self.content_length > 0:
                self.setRawMode()
                return
            elif self.file_upload:
                self._on_request_body(self.uploaded_file)
                self.file_upload = False
                self.uploaded_file = {}
                return

            self.request_callback(self._request)
        except Exception as exception:
            log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
            log.exception(exception)
            if self._request:
                self._request.finish()
            if self.transport:
                self.transport.loseConnection()
Exemple #11
0
def update_submission(store, id, request, finalize, language=GLSetting.memory_copy.default_language):

    context = store.find(Context, Context.id == unicode(request['context_gus'])).one()
    if not context:
        log.err("Context requested: [%s] not found!" % request['context_gus'])
        raise errors.ContextGusNotFound

    submission = store.find(InternalTip, InternalTip.id == unicode(id)).one()

    if not submission:

        log.debug("Creating a new submission in update!")
        submission = InternalTip()

        submission.escalation_threshold = context.escalation_threshold
        submission.access_limit = context.tip_max_access
        submission.download_limit = context.file_max_download
        submission.expiration_date = utc_future_date(seconds=context.tip_timetolive)
        submission.pertinence_counter = 0
        submission.context_id = context.id
        submission.creation_date = datetime_now()
        submission.mark = InternalTip._marker[0] # Submission

        try:
            store.add(submission)
        except Exception as excep:
            log.err("Storm/SQL Error: %s (update_submission)" % excep)
            raise errors.InternalServerError("Unable to commit on DB")

    # this may happen if a submission try to update a context
    if submission.context_id != context.id:
        log.err("Can't be changed context in a submission update")
        raise errors.ContextGusNotFound("Context are immutable")

    if submission.mark != InternalTip._marker[0]:
        log.err("Submission %s do not permit update (status %s)" % (id, submission.mark))
        raise errors.SubmissionConcluded

    files = request.get('files', [])
    try:
        import_files(store, submission, files, finalize)
    except Exception as excep:
        log.err("Submission update: files import fail: %s" % excep)
        log.exception(excep)
        raise excep

    wb_fields = request.get('wb_fields', [])
    try:
        fo = Fields(context.localized_fields, context.unique_fields)
        fo.validate_fields(wb_fields, strict_validation=finalize)
        submission.wb_fields = wb_fields
    except Exception as excep:
        log.err("Submission update: fields validation fail: %s" % excep)
        log.exception(excep)
        raise excep

    receivers = request.get('receivers', [])
    try:
        import_receivers(store, submission, receivers, required=finalize)
    except Exception as excep:
        log.err("Submission update: receiver import fail: %s" % excep)
        log.exception(excep)
        raise excep

    if finalize:
        submission.mark = InternalTip._marker[1] # Finalized

    submission_dict = wb_serialize_internaltip(submission)
    return submission_dict
Exemple #12
0
    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException(
                    "Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])
            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=self._remote_ip)

            try:
                self.content_length = int(headers.get("Content-Length", 0))
            except ValueError:
                raise _BadRequestException("Malformed Content-Length header")

            # we always use secure temporary files in case of large json or file uploads
            if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
                self._contentbuffer = StringIO('')
            else:
                self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)

            if headers.get("Expect") == "100-continue":
                self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")

            c_d_header = self._request.headers.get("Content-Disposition")
            if c_d_header is not None:
                key, pdict = parse_header(c_d_header)
                if key != 'attachment' or 'filename' not in pdict:
                    raise _BadRequestException("Malformed Content-Disposition header")

                self.file_upload = True
                self.uploaded_file['filename'] = pdict['filename']
                self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
                                                                               'application/octet-stream')

                self.uploaded_file['body'] = self._contentbuffer
                self.uploaded_file['body_len'] = int(self.content_length)
                self.uploaded_file['body_filepath'] = self._contentbuffer.filepath

            megabytes = int(self.content_length) / (1024 * 1024)

            if self.file_upload:
                limit_type = "upload"
                limit = GLSetting.memory_copy.maximum_filesize
            else:
                limit_type = "json"
                limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
                # is 1MB probably too high. probably this variable must be in kB

            # less than 1 megabytes is always accepted
            if megabytes > limit:
                log.err("Tried %s request larger than expected (%dMb > %dMb)" %
                        (limit_type,
                         megabytes,
                         limit))

                # In HTTP Protocol errors need to be managed differently than handlers
                raise errors.HTTPRawLimitReach

            if self.content_length > 0:
                self.setRawMode()
                return
            elif self.file_upload:
                self._on_request_body(self.uploaded_file)
                self.file_upload = False
                self.uploaded_file = {}
                return

            self.request_callback(self._request)
        except Exception as exception:
            log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
            log.exception(exception)
            if self._request:
                self._request.finish()
            if self.transport:
                self.transport.loseConnection()
Exemple #13
0
    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])
            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip
            )

            content_length = int(headers.get("Content-Length", 0))
            self.content_length = content_length

            if content_length:
                if headers.get("Expect") == "100-continue":
                    self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")

                if content_length < 100000:
                    self._contentbuffer = StringIO("")
                else:
                    self._contentbuffer = TemporaryFile()

                c_d_header = self._request.headers.get("Content-Disposition")
                if c_d_header is not None:
                    c_d_header = c_d_header.lower()
                    m = content_disposition_re.match(c_d_header)
                    if m is None:
                        raise Exception
                    self.file_upload = True
                    self.uploaded_file["filename"] = m.group(1)
                    self.uploaded_file["content_type"] = self._request.headers.get(
                        "Content-Type", "application/octet-stream"
                    )
                    self.uploaded_file["body"] = self._contentbuffer
                    self.uploaded_file["body_len"] = int(content_length)

                megabytes = int(content_length) / (1024 * 1024)

                if self.file_upload:
                    limit_type = "upload"
                    limit = GLSetting.memory_copy.maximum_filesize
                else:
                    limit_type = "json"
                    limit = 1000000  # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
                    # is 1MB probably too high. probably this variable must be
                    # in kB

                # less than 1 megabytes is always accepted
                if megabytes > limit:

                    log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit))

                    # In HTTP Protocol errors need to be managed differently than handlers
                    raise errors.HTTPRawLimitReach

                self.setRawMode()
                return

            self.request_callback(self._request)
        except Exception as exception:
            log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
            log.exception(exception)
            if self._request:
                self._request.finish()
            if self.transport:
                self.transport.loseConnection()