def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith(
"application/x-www-form-urlencoded"
) and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
arguments = parse_qs_bytes(native_str(self._request.body))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self._request.arguments.setdefault(
name, []).extend(values)
elif content_type.startswith(
"application/x-www-form-urlencoded"):
raise errors.InvalidInputFormat(
"content type application/x-www-form-urlencoded not supported"
)
elif content_type.startswith("multipart/form-data"):
raise errors.InvalidInputFormat(
"content type multipart/form-data not supported")
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" %
(self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded"):
raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
elif content_type.startswith("multipart/form-data"):
raise errors.InvalidInputFormat("content type multipart/form-data not supported")
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def db_update_submission(store, submission_id, request, finalize, language):
context = store.find(Context,
Context.id == unicode(request['context_id'])).one()
if not context:
log.err("Context requested: [%s] not found!" % request['context_id'])
raise errors.ContextIdNotFound
submission = store.find(InternalTip,
InternalTip.id == unicode(submission_id)).one()
if not submission:
log.err("Invalid Submission requested %s in PUT" % submission_id)
raise errors.SubmissionIdNotFound
# this may happen if a submission try to update a context
if submission.context_id != context.id:
log.err("Can't be changed context in a submission update")
raise errors.ContextIdNotFound()
if submission.mark != u'submission':
log.err("Submission %s do not permit update (status %s)" %
(submission_id, submission.mark))
raise errors.SubmissionConcluded
try:
import_files(store, submission, request['files'])
except Exception as excep:
log.err("Submission update: files import fail: %s" % excep)
log.exception(excep)
raise excep
try:
wb_steps = request['wb_steps']
if finalize:
steps = db_get_context_steps(store, context.id, language)
verify_steps(steps, wb_steps)
submission.wb_steps = wb_steps
except Exception as excep:
log.err("Submission update: fields validation fail: %s" % excep)
log.exception(excep)
raise excep
try:
import_receivers(store,
submission,
request['receivers'],
required=finalize)
except Exception as excep:
log.err("Submission update: receiver import fail: %s" % excep)
log.exception(excep)
raise excep
if finalize:
submission.mark = u'finalize' # Finalized
else:
submission.mark = u'submission' # Submission
submission_dict = wb_serialize_internaltip(submission)
return submission_dict
def _on_request_body(self, data):
try:
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PATCH", "PUT"):
if content_type.startswith("application/x-www-form-urlencoded") and self.content_length < GLSetting.www_form_urlencoded_maximum_size:
arguments = parse_qs_bytes(native_str(self._request.body))
for name, values in arguments.iteritems():
values = [v for v in values if v]
if values:
self._request.arguments.setdefault(name,
[]).extend(values)
elif content_type.startswith("application/x-www-form-urlencoded"):
raise errors.InvalidInputFormat("content type application/x-www-form-urlencoded not supported")
elif content_type.startswith("multipart/form-data"):
raise errors.InvalidInputFormat("content type multipart/form-data not supported")
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language):
context = store.find(Context, Context.id == unicode(request['context_id'])).one()
if not context:
log.err("Context requested: [%s] not found!" % request['context_id'])
raise errors.ContextIdNotFound
submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one()
if not submission:
log.err("Invalid Submission requested %s in PUT" % submission_id)
raise errors.SubmissionIdNotFound
# this may happen if a submission try to update a context
if submission.context_id != context.id:
log.err("Can't be changed context in a submission update")
raise errors.ContextIdNotFound("Context are immutable")
if submission.mark != u'submission':
log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark))
raise errors.SubmissionConcluded
try:
import_files(store, submission, request['files'], finalize)
except Exception as excep:
log.err("Submission update: files import fail: %s" % excep)
log.exception(excep)
raise excep
try:
wb_steps = request['wb_steps']
if finalize:
steps = db_get_context_steps(store, context.id, language)
verify_steps(steps, wb_steps)
submission.wb_steps = wb_steps
except Exception as excep:
log.err("Submission update: fields validation fail: %s" % excep)
log.exception(excep)
raise excep
try:
import_receivers(store, submission, request['receivers'], required=finalize)
except Exception as excep:
log.err("Submission update: receiver import fail: %s" % excep)
log.exception(excep)
raise excep
if finalize:
submission.mark = u'finalize' # Finalized
else:
submission.mark = u'submission' # Submission
submission_dict = wb_serialize_internaltip(submission)
return submission_dict
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language):
context = store.find(Context, Context.id == unicode(request["context_id"])).one()
if not context:
log.err("Context requested: [%s] not found!" % request["context_id"])
raise errors.ContextIdNotFound
submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one()
if not submission:
log.err("Invalid Submission requested %s in PUT" % submission_id)
raise errors.SubmissionIdNotFound
# this may happen if a submission try to update a context
if submission.context_id != context.id:
log.err("Can't be changed context in a submission update")
raise errors.ContextIdNotFound("Context are immutable")
if submission.mark != InternalTip._marker[0]:
log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark))
raise errors.SubmissionConcluded
try:
import_files(store, submission, request["files"], finalize)
except Exception as excep:
log.err("Submission update: files import fail: %s" % excep)
log.exception(excep)
raise excep
try:
wb_fields = request["wb_fields"]
fo = Fields(context.localized_fields, context.unique_fields)
fo.validate_fields(wb_fields, language, strict_validation=finalize)
submission.wb_fields = wb_fields
except Exception as excep:
log.err("Submission update: fields validation fail: %s" % excep)
log.exception(excep)
raise excep
try:
import_receivers(store, submission, request["receivers"], required=finalize)
except Exception as excep:
log.err("Submission update: receiver import fail: %s" % excep)
log.exception(excep)
raise excep
if finalize:
submission.mark = InternalTip._marker[1] # Finalized
submission_dict = wb_serialize_internaltip(submission)
return submission_dict
def update_submission(store, submission_id, request, finalize, language=GLSetting.memory_copy.default_language):
context = store.find(Context, Context.id == unicode(request['context_id'])).one()
if not context:
log.err("Context requested: [%s] not found!" % request['context_id'])
raise errors.ContextIdNotFound
submission = store.find(InternalTip, InternalTip.id == unicode(submission_id)).one()
if not submission:
log.err("Invalid Submission requested %s in PUT" % submission_id)
raise errors.SubmissionIdNotFound
# this may happen if a submission try to update a context
if submission.context_id != context.id:
log.err("Can't be changed context in a submission update")
raise errors.ContextIdNotFound("Context are immutable")
if submission.mark != InternalTip._marker[0]:
log.err("Submission %s do not permit update (status %s)" % (submission_id, submission.mark))
raise errors.SubmissionConcluded
try:
import_files(store, submission, request['files'], finalize)
except Exception as excep:
log.err("Submission update: files import fail: %s" % excep)
log.exception(excep)
raise excep
try:
wb_fields = request['wb_fields']
fo = Fields(context.localized_fields, context.unique_fields)
fo.validate_fields(wb_fields, language, strict_validation=finalize)
submission.wb_fields = wb_fields
except Exception as excep:
log.err("Submission update: fields validation fail: %s" % excep)
log.exception(excep)
raise excep
try:
import_receivers(store, submission, request['receivers'], required=finalize)
except Exception as excep:
log.err("Submission update: receiver import fail: %s" % excep)
log.exception(excep)
raise excep
if finalize:
submission.mark = InternalTip._marker[1] # Finalized
submission_dict = wb_serialize_internaltip(submission)
return submission_dict
def on_error(self, excep):
log.err("Exception while running %s" % self.name)
log.exception(excep)
extract_exception_traceback_and_schedule_email(excep)
def on_error(self, excep):
error = "Job %s died with runtime %.4f [low: %.4f, high: %.4f]" % \
(self.name, self.mean_time, self.low_time, self.high_time)
log.err(error)
log.exception(excep)
extract_exception_traceback_and_schedule_email(excep)
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
try:
self.content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
# we always use secure temporary files in case of large json or file uploads
if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
self._contentbuffer = StringIO('')
else:
self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
key, pdict = parse_header(c_d_header)
if key != 'attachment' or 'filename' not in pdict:
raise _BadRequestException("Malformed Content-Disposition header")
self.file_upload = True
self.uploaded_file['filename'] = pdict['filename']
self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
'application/octet-stream')
self.uploaded_file['body'] = self._contentbuffer
self.uploaded_file['body_len'] = int(self.content_length)
self.uploaded_file['body_filepath'] = self._contentbuffer.filepath
megabytes = int(self.content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" %
(limit_type,
megabytes,
limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
if self.content_length > 0:
self.setRawMode()
return
elif self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def update_submission(store, id, request, finalize, language=GLSetting.memory_copy.default_language):
context = store.find(Context, Context.id == unicode(request['context_gus'])).one()
if not context:
log.err("Context requested: [%s] not found!" % request['context_gus'])
raise errors.ContextGusNotFound
submission = store.find(InternalTip, InternalTip.id == unicode(id)).one()
if not submission:
log.debug("Creating a new submission in update!")
submission = InternalTip()
submission.escalation_threshold = context.escalation_threshold
submission.access_limit = context.tip_max_access
submission.download_limit = context.file_max_download
submission.expiration_date = utc_future_date(seconds=context.tip_timetolive)
submission.pertinence_counter = 0
submission.context_id = context.id
submission.creation_date = datetime_now()
submission.mark = InternalTip._marker[0] # Submission
try:
store.add(submission)
except Exception as excep:
log.err("Storm/SQL Error: %s (update_submission)" % excep)
raise errors.InternalServerError("Unable to commit on DB")
# this may happen if a submission try to update a context
if submission.context_id != context.id:
log.err("Can't be changed context in a submission update")
raise errors.ContextGusNotFound("Context are immutable")
if submission.mark != InternalTip._marker[0]:
log.err("Submission %s do not permit update (status %s)" % (id, submission.mark))
raise errors.SubmissionConcluded
files = request.get('files', [])
try:
import_files(store, submission, files, finalize)
except Exception as excep:
log.err("Submission update: files import fail: %s" % excep)
log.exception(excep)
raise excep
wb_fields = request.get('wb_fields', [])
try:
fo = Fields(context.localized_fields, context.unique_fields)
fo.validate_fields(wb_fields, strict_validation=finalize)
submission.wb_fields = wb_fields
except Exception as excep:
log.err("Submission update: fields validation fail: %s" % excep)
log.exception(excep)
raise excep
receivers = request.get('receivers', [])
try:
import_receivers(store, submission, receivers, required=finalize)
except Exception as excep:
log.err("Submission update: receiver import fail: %s" % excep)
log.exception(excep)
raise excep
if finalize:
submission.mark = InternalTip._marker[1] # Finalized
submission_dict = wb_serialize_internaltip(submission)
return submission_dict
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException(
"Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version,
headers=headers, remote_ip=self._remote_ip)
try:
self.content_length = int(headers.get("Content-Length", 0))
except ValueError:
raise _BadRequestException("Malformed Content-Length header")
# we always use secure temporary files in case of large json or file uploads
if self.content_length < 100000 and self._request.headers.get("Content-Disposition") is None:
self._contentbuffer = StringIO('')
else:
self._contentbuffer = GLSecureTemporaryFile(GLSetting.tmp_upload_path)
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
key, pdict = parse_header(c_d_header)
if key != 'attachment' or 'filename' not in pdict:
raise _BadRequestException("Malformed Content-Disposition header")
self.file_upload = True
self.uploaded_file['filename'] = pdict['filename']
self.uploaded_file['content_type'] = self._request.headers.get("Content-Type",
'application/octet-stream')
self.uploaded_file['body'] = self._contentbuffer
self.uploaded_file['body_len'] = int(self.content_length)
self.uploaded_file['body_filepath'] = self._contentbuffer.filepath
megabytes = int(self.content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" %
(limit_type,
megabytes,
limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
if self.content_length > 0:
self.setRawMode()
return
elif self.file_upload:
self._on_request_body(self.uploaded_file)
self.file_upload = False
self.uploaded_file = {}
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
def _on_headers(self, data):
try:
data = native_str(data.decode("latin1"))
eol = data.find("\r\n")
start_line = data[:eol]
try:
method, uri, version = start_line.split(" ")
except ValueError:
raise _BadRequestException("Malformed HTTP request line")
if not version.startswith("HTTP/"):
raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
headers = httputil.HTTPHeaders.parse(data[eol:])
self._request = HTTPRequest(
connection=self, method=method, uri=uri, version=version, headers=headers, remote_ip=self._remote_ip
)
content_length = int(headers.get("Content-Length", 0))
self.content_length = content_length
if content_length:
if headers.get("Expect") == "100-continue":
self.transport.write("HTTP/1.1 100 (Continue)\r\n\r\n")
if content_length < 100000:
self._contentbuffer = StringIO("")
else:
self._contentbuffer = TemporaryFile()
c_d_header = self._request.headers.get("Content-Disposition")
if c_d_header is not None:
c_d_header = c_d_header.lower()
m = content_disposition_re.match(c_d_header)
if m is None:
raise Exception
self.file_upload = True
self.uploaded_file["filename"] = m.group(1)
self.uploaded_file["content_type"] = self._request.headers.get(
"Content-Type", "application/octet-stream"
)
self.uploaded_file["body"] = self._contentbuffer
self.uploaded_file["body_len"] = int(content_length)
megabytes = int(content_length) / (1024 * 1024)
if self.file_upload:
limit_type = "upload"
limit = GLSetting.memory_copy.maximum_filesize
else:
limit_type = "json"
limit = 1000000 # 1MB fixme: add GLSetting.memory_copy.maximum_jsonsize
# is 1MB probably too high. probably this variable must be
# in kB
# less than 1 megabytes is always accepted
if megabytes > limit:
log.err("Tried %s request larger than expected (%dMb > %dMb)" % (limit_type, megabytes, limit))
# In HTTP Protocol errors need to be managed differently than handlers
raise errors.HTTPRawLimitReach
self.setRawMode()
return
self.request_callback(self._request)
except Exception as exception:
log.msg("Malformed HTTP request from %s: %s" % (self._remote_ip, exception))
log.exception(exception)
if self._request:
self._request.finish()
if self.transport:
self.transport.loseConnection()
