def post_submission(request, draft):
with open(draft.get_file_name()) as file:
wrapper = Draft(file.read().decode('utf8'), file.name)
submission = Submission(
name=draft.name,
title=draft.title,
rev=draft.rev,
pages=draft.pages,
file_size=os.path.getsize(draft.get_file_name()),
document_date=wrapper.get_creation_date(),
submission_date=datetime.date.today(),
group_id=draft.group.id,
remote_ip=request.META['REMOTE_ADDR'],
first_two_pages=''.join(wrapper.pages[:2]),
state=DraftSubmissionStateName.objects.get(slug="posted"),
abstract=draft.abstract,
file_types=','.join(file_types_for_draft(draft)),
)
submission.save()
SubmissionEvent.objects.create(submission=submission,
by=request.user.person,
desc="Submitted and posted manually")
return submission
def process_files(request,draft):
'''
This function takes a request object and draft object.
It obtains the list of file objects (ie from request.FILES), uploads
the files by calling handle_file_upload() and returns
the basename, revision number and a list of file types. Basename and revision
are assumed to be the same for all because this is part of the validation process.
It also creates the IdSubmissionDetail record WITHOUT saving, and places it in the
session for saving in the final action step.
'''
files = request.FILES
file = files[files.keys()[0]]
filename = os.path.splitext(file.name)[0]
revision = os.path.splitext(file.name)[0][-2:]
basename = get_base(filename)
file_type_list = []
for file in files.values():
extension = os.path.splitext(file.name)[1]
file_type_list.append(extension)
if extension == '.txt':
txt_size = file.size
wrapper = Draft(file.read(),file.name)
handle_uploaded_file(file)
# create IdSubmissionDetail record, leaved unsaved
idsub = IdSubmissionDetail(
id_document_name=draft.title,
filename=basename,
revision=revision,
txt_page_count=draft.pages,
filesize=txt_size,
creation_date=wrapper.get_creation_date(),
submission_date=datetime.date.today(),
idnits_message='idnits bypassed by manual posting',
temp_id_document_tag=None,
group_acronym_id=draft.group.id,
remote_ip=request.META['REMOTE_ADDR'],
first_two_pages=''.join(wrapper.pages[:2]),
status_id=-2,
abstract=draft.abstract,
file_type=','.join(file_type_list),
man_posted_date=datetime.date.today(),
man_posted_by=request.user.get_profile())
request.session['idsub'] = idsub
return (filename,revision,file_type_list)
def process_files(request, draft):
'''
This function takes a request object and draft object.
It obtains the list of file objects (ie from request.FILES), uploads
the files by calling handle_file_upload() and returns
the basename, revision number and a list of file types. Basename and revision
are assumed to be the same for all because this is part of the validation process.
It also creates the IdSubmissionDetail record WITHOUT saving, and places it in the
session for saving in the final action step.
'''
files = request.FILES
file = files[files.keys()[0]]
filename = os.path.splitext(file.name)[0]
revision = os.path.splitext(file.name)[0][-2:]
basename = get_base(filename)
file_type_list = []
for file in files.values():
extension = os.path.splitext(file.name)[1]
file_type_list.append(extension)
if extension == '.txt':
txt_size = file.size
wrapper = Draft(file.read(), file.name)
handle_uploaded_file(file)
# create IdSubmissionDetail record, leaved unsaved
idsub = IdSubmissionDetail(
id_document_name=draft.title,
filename=basename,
revision=revision,
txt_page_count=draft.pages,
filesize=txt_size,
creation_date=wrapper.get_creation_date(),
submission_date=datetime.date.today(),
idnits_message='idnits bypassed by manual posting',
temp_id_document_tag=None,
group_acronym_id=draft.group.id,
remote_ip=request.META['REMOTE_ADDR'],
first_two_pages=''.join(wrapper.pages[:2]),
status_id=-2,
abstract=draft.abstract,
file_type=','.join(file_type_list),
man_posted_date=datetime.date.today(),
man_posted_by=request.user.get_profile())
request.session['idsub'] = idsub
return (filename, revision, file_type_list)
def get_draft(self):
if self.draft:
return self.draft
txt_file = self.cleaned_data['txt']
txt_file.seek(0)
self.draft = Draft(txt_file.read(), txt_file.name)
txt_file.seek(0)
return self.draft
class SubmissionBaseUploadForm(forms.Form):
xml = forms.FileField(label=u'.xml format', required=True)
def __init__(self, request, *args, **kwargs):
super(SubmissionBaseUploadForm, self).__init__(*args, **kwargs)
self.remote_ip = request.META.get('REMOTE_ADDR', None)
self.request = request
self.in_first_cut_off = False
self.cutoff_warning = ""
self.shutdown = False
self.set_cutoff_warnings()
self.group = None
self.filename = None
self.revision = None
self.title = None
self.abstract = None
self.authors = []
self.parsed_draft = None
self.file_types = []
# No code currently (14 Sep 2017) uses this class directly; it is
# only used through its subclasses. The two assignments below are
# set to trigger an exception if it is used directly only to make
# sure that adequate consideration is made if it is decided to use it
# directly in the future. Feel free to set these appropriately to
# avoid the exceptions in that case:
self.formats = None # None will raise an exception in clean() if this isn't changed in a subclass
self.base_formats = None # None will raise an exception in clean() if this isn't changed in a subclass
def set_cutoff_warnings(self):
now = datetime.datetime.now(pytz.utc)
meeting = Meeting.get_current_meeting()
#
cutoff_00 = meeting.get_00_cutoff()
cutoff_01 = meeting.get_01_cutoff()
reopen = meeting.get_reopen_time()
#
cutoff_00_str = cutoff_00.strftime("%Y-%m-%d %H:%M %Z")
cutoff_01_str = cutoff_01.strftime("%Y-%m-%d %H:%M %Z")
reopen_str = reopen.strftime("%Y-%m-%d %H:%M %Z")
if cutoff_00 == cutoff_01:
if now.date() >= (cutoff_00.date() -
meeting.idsubmit_cutoff_warning_days
) and now.date() < cutoff_00.date():
self.cutoff_warning = (
'The last submission time for Internet-Drafts before %s is %s.<br/><br/>'
% (meeting, cutoff_00_str))
elif now <= cutoff_00:
self.cutoff_warning = (
'The last submission time for new Internet-Drafts before the meeting is %s.<br/>'
'After that, you will not be able to submit drafts until after %s (IETF-meeting local time)'
% (
cutoff_00_str,
reopen_str,
))
else:
if now.date() >= (cutoff_00.date() -
meeting.idsubmit_cutoff_warning_days
) and now.date() < cutoff_00.date():
self.cutoff_warning = (
'The last submission time for new documents (i.e., version -00 Internet-Drafts) before %s is %s.<br/><br/>'
% (meeting, cutoff_00_str) +
'The last submission time for revisions to existing documents before %s is %s.<br/>'
% (meeting, cutoff_01_str))
elif now.date() >= cutoff_00.date() and now <= cutoff_01:
# We are in the first_cut_off
if now < cutoff_00:
self.cutoff_warning = (
'The last submission time for new documents (i.e., version -00 Internet-Drafts) before the meeting is %s.<br/>'
'After that, you will not be able to submit a new document until after %s (IETF-meeting local time)'
% (
cutoff_00_str,
reopen_str,
))
else: # No 00 version allowed
self.cutoff_warning = (
'The last submission time for new documents (i.e., version -00 Internet-Drafts) was %s.<br/>'
'You will not be able to submit a new document until after %s (IETF-meeting local time).<br/><br>'
'You can still submit a version -01 or higher Internet-Draft until %s'
% (
cutoff_00_str,
reopen_str,
cutoff_01_str,
))
self.in_first_cut_off = True
if now > cutoff_01 and now < reopen:
self.cutoff_warning = (
'The last submission time for the I-D submission was %s.<br/><br>'
'The I-D submission tool will be reopened after %s (IETF-meeting local time).'
% (cutoff_01_str, reopen_str))
self.shutdown = True
def clean_file(self, field_name, parser_class):
f = self.cleaned_data[field_name]
if not f:
return f
parsed_info = parser_class(f).critical_parse()
if parsed_info.errors:
raise forms.ValidationError(parsed_info.errors)
return f
def clean_xml(self):
return self.clean_file("xml", XMLParser)
def clean(self):
if self.shutdown and not has_role(self.request.user, "Secretariat"):
raise forms.ValidationError(
'The submission tool is currently shut down')
for ext in self.formats:
f = self.cleaned_data.get(ext, None)
if not f:
continue
self.file_types.append('.%s' % ext)
if not ('.txt' in self.file_types or '.xml' in self.file_types):
raise forms.ValidationError(
'Unexpected submission file types; found %s, but %s is required'
% (', '.join(self.file_types), ' or '.join(self.base_formats)))
#debug.show('self.cleaned_data["xml"]')
if self.cleaned_data.get('xml'):
#if not self.cleaned_data.get('txt'):
xml_file = self.cleaned_data.get('xml')
name, ext = os.path.splitext(os.path.basename(xml_file.name))
tfh, tfn = tempfile.mkstemp(prefix=name + '-', suffix='.xml')
try:
# We need to write the xml file to disk in order to hand it
# over to the xml parser. XXX FIXME: investigate updating
# xml2rfc to be able to work with file handles to in-memory
# files.
with open(tfn, 'wb+') as tf:
for chunk in xml_file.chunks():
tf.write(chunk)
os.environ["XML_LIBRARY"] = settings.XML_LIBRARY
try:
parser = xml2rfc.XmlRfcParser(str(tfn), quiet=True)
self.xmltree = parser.parse()
ok, errors = self.xmltree.validate()
except Exception as exc:
raise forms.ValidationError(
"An exception occurred when trying to process the XML file: %s"
% exc.msg)
if not ok:
# Each error has properties:
#
# message: the message text
# domain: the domain ID (see lxml.etree.ErrorDomains)
# type: the message type ID (see lxml.etree.ErrorTypes)
# level: the log level ID (see lxml.etree.ErrorLevels)
# line: the line at which the message originated (if applicable)
# column: the character column at which the message originated (if applicable)
# filename: the name of the file in which the message originated (if applicable)
raise forms.ValidationError([
forms.ValidationError(
"One or more XML validation errors occurred when processing the XML file:"
)
] + [
forms.ValidationError("%s: Line %s: %s" %
(xml_file.name, e.line,
e.message),
code="%s" % e.type)
for e in errors
])
self.xmlroot = self.xmltree.getroot()
draftname = self.xmlroot.attrib.get('docName')
if draftname is None:
raise forms.ValidationError(
"No docName attribute found in the xml root element")
revmatch = re.search("-[0-9][0-9]$", draftname)
if revmatch:
self.revision = draftname[-2:]
self.filename = draftname[:-3]
else:
self.revision = None
self.filename = draftname
self.title = self.xmlroot.findtext('front/title').strip()
if type(self.title) is unicode:
self.title = unidecode(self.title)
self.abstract = self.xmlroot.findtext('front/abstract').strip()
if type(self.abstract) is unicode:
self.abstract = unidecode(self.abstract)
author_info = self.xmlroot.findall('front/author')
for author in author_info:
info = {
"name": author.attrib.get('fullname'),
"email": author.findtext('address/email'),
"affiliation": author.findtext('organization'),
"country": author.findtext('address/postal/country'),
}
for item in info:
if info[item]:
info[item] = info[item].strip()
self.authors.append(info)
except forms.ValidationError:
raise
finally:
os.close(tfh)
os.unlink(tfn)
if self.cleaned_data.get('txt'):
# try to parse it
txt_file = self.cleaned_data['txt']
txt_file.seek(0)
bytes = txt_file.read()
txt_file.seek(0)
try:
text = bytes.decode('utf8')
except UnicodeDecodeError as e:
raise forms.ValidationError(
'Failed decoding the uploaded file: "%s"' % str(e))
#
self.parsed_draft = Draft(text, txt_file.name)
self.filename = self.parsed_draft.filename
self.revision = self.parsed_draft.revision
self.title = self.parsed_draft.get_title()
if not self.filename:
raise forms.ValidationError(
"Could not extract a valid draft name from the upload"
"To fix this in a text upload, please make sure that the full draft name including "
"revision number appears centered on its own line below the document title on the "
"first page. In an xml upload, please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full draft name including "
"revision number.")
if not self.revision:
raise forms.ValidationError(
"Could not extract a valid draft revision from the upload. "
"To fix this in a text upload, please make sure that the full draft name including "
"revision number appears centered on its own line below the document title on the "
"first page. In an xml upload, please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full draft name including "
"revision number.")
if not self.title:
raise forms.ValidationError(
"Could not extract a valid title from the upload")
if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'):
# check group
self.group = self.deduce_group()
# check existing
existing = Submission.objects.filter(
name=self.filename,
rev=self.revision).exclude(state__in=("posted", "cancel",
"waiting-for-draft"))
if existing:
raise forms.ValidationError(
mark_safe(
'A submission with same name and revision is currently being processed. <a href="%s">Check the status here.</a>'
%
urlreverse("ietf.submit.views.submission_status",
kwargs={'submission_id': existing[0].pk})))
# cut-off
if self.revision == '00' and self.in_first_cut_off:
raise forms.ValidationError(mark_safe(self.cutoff_warning))
# check thresholds
today = datetime.date.today()
self.check_submissions_tresholds(
"for the draft %s" % self.filename,
dict(name=self.filename,
rev=self.revision,
submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME,
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
)
self.check_submissions_tresholds(
"for the same submitter",
dict(remote_ip=self.remote_ip, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER,
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
)
if self.group:
self.check_submissions_tresholds(
"for the group \"%s\"" % (self.group.acronym),
dict(group=self.group, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP,
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE,
)
self.check_submissions_tresholds(
"across all submitters",
dict(submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS,
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
return super(SubmissionBaseUploadForm, self).clean()
def check_submissions_tresholds(self, which, filter_kwargs, max_amount,
max_size):
submissions = Submission.objects.filter(**filter_kwargs)
if len(submissions) > max_amount:
raise forms.ValidationError(
"Max submissions %s has been reached for today (maximum is %s submissions)."
% (which, max_amount))
if sum(s.file_size
for s in submissions if s.file_size) > max_size * 1024 * 1024:
raise forms.ValidationError(
"Max uploaded amount %s has been reached for today (maximum is %s MB)."
% (which, max_size))
def deduce_group(self):
"""Figure out group from name or previously submitted draft, returns None if individual."""
name = self.filename
existing_draft = Document.objects.filter(name=name, type="draft")
if existing_draft:
group = existing_draft[0].group
if group and group.type_id not in ("individ", "area"):
return group
else:
return None
else:
if name.startswith('draft-ietf-') or name.startswith(
"draft-irtf-"):
components = name.split("-")
if len(components) < 3:
raise forms.ValidationError(
u"The draft name \"%s\" is missing a third part, please rename it"
% name)
if components[1] == "ietf":
group_type = "wg"
elif components[1] == "irtf":
group_type = "rg"
# first check groups with dashes
for g in Group.objects.filter(acronym__contains="-",
type=group_type):
if name.startswith('draft-%s-%s-' %
(components[1], g.acronym)):
return g
try:
return Group.objects.get(acronym=components[2],
type=group_type)
except Group.DoesNotExist:
raise forms.ValidationError(
'There is no active group with acronym \'%s\', please rename your draft'
% components[2])
elif name.startswith("draft-rfc-"):
return Group.objects.get(acronym="iesg")
elif name.startswith("draft-irtf-"):
return Group.objects.get(acronym="irtf")
elif name.startswith("draft-iab-"):
return Group.objects.get(acronym="iab")
elif name.startswith("draft-iana-"):
return Group.objects.get(acronym="iana")
elif name.startswith("draft-rfc-editor-") or name.startswith(
"draft-rfced-") or name.startswith("draft-rfceditor-"):
return Group.objects.get(acronym="rfceditor")
else:
return None
def clean(self):
if self.shutdown and not has_role(self.request.user, "Secretariat"):
raise forms.ValidationError(
'The submission tool is currently shut down')
for ext in self.formats:
f = self.cleaned_data.get(ext, None)
if not f:
continue
self.file_types.append('.%s' % ext)
if not ('.txt' in self.file_types or '.xml' in self.file_types):
raise forms.ValidationError(
'Unexpected submission file types; found %s, but %s is required'
% (', '.join(self.file_types), ' or '.join(self.base_formats)))
#debug.show('self.cleaned_data["xml"]')
if self.cleaned_data.get('xml'):
#if not self.cleaned_data.get('txt'):
xml_file = self.cleaned_data.get('xml')
name, ext = os.path.splitext(os.path.basename(xml_file.name))
tfh, tfn = tempfile.mkstemp(prefix=name + '-', suffix='.xml')
try:
# We need to write the xml file to disk in order to hand it
# over to the xml parser. XXX FIXME: investigate updating
# xml2rfc to be able to work with file handles to in-memory
# files.
with open(tfn, 'wb+') as tf:
for chunk in xml_file.chunks():
tf.write(chunk)
os.environ["XML_LIBRARY"] = settings.XML_LIBRARY
try:
parser = xml2rfc.XmlRfcParser(str(tfn), quiet=True)
self.xmltree = parser.parse()
ok, errors = self.xmltree.validate()
except Exception as exc:
raise forms.ValidationError(
"An exception occurred when trying to process the XML file: %s"
% exc.msg)
if not ok:
# Each error has properties:
#
# message: the message text
# domain: the domain ID (see lxml.etree.ErrorDomains)
# type: the message type ID (see lxml.etree.ErrorTypes)
# level: the log level ID (see lxml.etree.ErrorLevels)
# line: the line at which the message originated (if applicable)
# column: the character column at which the message originated (if applicable)
# filename: the name of the file in which the message originated (if applicable)
raise forms.ValidationError([
forms.ValidationError(
"One or more XML validation errors occurred when processing the XML file:"
)
] + [
forms.ValidationError("%s: Line %s: %s" %
(xml_file.name, e.line,
e.message),
code="%s" % e.type)
for e in errors
])
self.xmlroot = self.xmltree.getroot()
draftname = self.xmlroot.attrib.get('docName')
if draftname is None:
raise forms.ValidationError(
"No docName attribute found in the xml root element")
revmatch = re.search("-[0-9][0-9]$", draftname)
if revmatch:
self.revision = draftname[-2:]
self.filename = draftname[:-3]
else:
self.revision = None
self.filename = draftname
self.title = self.xmlroot.findtext('front/title').strip()
if type(self.title) is unicode:
self.title = unidecode(self.title)
self.abstract = self.xmlroot.findtext('front/abstract').strip()
if type(self.abstract) is unicode:
self.abstract = unidecode(self.abstract)
author_info = self.xmlroot.findall('front/author')
for author in author_info:
info = {
"name": author.attrib.get('fullname'),
"email": author.findtext('address/email'),
"affiliation": author.findtext('organization'),
"country": author.findtext('address/postal/country'),
}
for item in info:
if info[item]:
info[item] = info[item].strip()
self.authors.append(info)
except forms.ValidationError:
raise
finally:
os.close(tfh)
os.unlink(tfn)
if self.cleaned_data.get('txt'):
# try to parse it
txt_file = self.cleaned_data['txt']
txt_file.seek(0)
bytes = txt_file.read()
txt_file.seek(0)
try:
text = bytes.decode('utf8')
except UnicodeDecodeError as e:
raise forms.ValidationError(
'Failed decoding the uploaded file: "%s"' % str(e))
#
self.parsed_draft = Draft(text, txt_file.name)
self.filename = self.parsed_draft.filename
self.revision = self.parsed_draft.revision
self.title = self.parsed_draft.get_title()
if not self.filename:
raise forms.ValidationError(
"Could not extract a valid draft name from the upload"
"To fix this in a text upload, please make sure that the full draft name including "
"revision number appears centered on its own line below the document title on the "
"first page. In an xml upload, please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full draft name including "
"revision number.")
if not self.revision:
raise forms.ValidationError(
"Could not extract a valid draft revision from the upload. "
"To fix this in a text upload, please make sure that the full draft name including "
"revision number appears centered on its own line below the document title on the "
"first page. In an xml upload, please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full draft name including "
"revision number.")
if not self.title:
raise forms.ValidationError(
"Could not extract a valid title from the upload")
if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'):
# check group
self.group = self.deduce_group()
# check existing
existing = Submission.objects.filter(
name=self.filename,
rev=self.revision).exclude(state__in=("posted", "cancel",
"waiting-for-draft"))
if existing:
raise forms.ValidationError(
mark_safe(
'A submission with same name and revision is currently being processed. <a href="%s">Check the status here.</a>'
%
urlreverse("ietf.submit.views.submission_status",
kwargs={'submission_id': existing[0].pk})))
# cut-off
if self.revision == '00' and self.in_first_cut_off:
raise forms.ValidationError(mark_safe(self.cutoff_warning))
# check thresholds
today = datetime.date.today()
self.check_submissions_tresholds(
"for the draft %s" % self.filename,
dict(name=self.filename,
rev=self.revision,
submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME,
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
)
self.check_submissions_tresholds(
"for the same submitter",
dict(remote_ip=self.remote_ip, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER,
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
)
if self.group:
self.check_submissions_tresholds(
"for the group \"%s\"" % (self.group.acronym),
dict(group=self.group, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP,
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE,
)
self.check_submissions_tresholds(
"across all submitters",
dict(submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS,
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
return super(SubmissionBaseUploadForm, self).clean()
def clean(self):
if self.shutdown:
raise forms.ValidationError('The tool is shut down')
# sanity check that paths exist (for development servers)
for s in ("IDSUBMIT_STAGING_PATH", "IDSUBMIT_IDNITS_BINARY",
"IDSUBMIT_REPOSITORY_PATH", "INTERNET_DRAFT_ARCHIVE_DIR"):
if not os.path.exists(getattr(settings, s)):
raise forms.ValidationError('%s defined in settings.py does not exist' % s)
if self.cleaned_data.get('txt'):
# try to parse it
txt_file = self.cleaned_data['txt']
txt_file.seek(0)
self.parsed_draft = Draft(txt_file.read(), txt_file.name)
txt_file.seek(0)
if not self.parsed_draft.filename:
raise forms.ValidationError("Draft parser could not extract a valid draft name from the .txt file")
if not self.parsed_draft.get_title():
raise forms.ValidationError("Draft parser could not extract a valid title from the .txt file")
# check group
self.group = self.deduce_group()
# check existing
existing = Submission.objects.filter(name=self.parsed_draft.filename, rev=self.parsed_draft.revision).exclude(state__in=("posted", "cancel"))
if existing:
raise forms.ValidationError(mark_safe('Submission with same name and revision is currently being processed. <a href="%s">Check the status here</a>' % urlreverse("submit_submission_status", kwargs={ 'submission_id': existing[0].pk })))
# cut-off
if self.parsed_draft.revision == '00' and self.in_first_cut_off:
raise forms.ValidationError(mark_safe(self.cutoff_warning))
# check thresholds
today = datetime.date.today()
self.check_submissions_tresholds(
"for the draft %s" % self.parsed_draft.filename,
dict(name=self.parsed_draft.filename, rev=self.parsed_draft.revision, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
)
self.check_submissions_tresholds(
"for the same submitter",
dict(remote_ip=self.remote_ip, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER, settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
)
if self.group:
self.check_submissions_tresholds(
"for the group \"%s\"" % (self.group.acronym),
dict(group=self.group, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP, settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE,
)
self.check_submissions_tresholds(
"across all submitters",
dict(submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
return super(UploadForm, self).clean()
class UploadForm(forms.Form):
txt = forms.FileField(label=u'.txt format', required=True)
xml = forms.FileField(label=u'.xml format', required=False)
pdf = forms.FileField(label=u'.pdf format', required=False)
ps = forms.FileField(label=u'.ps format', required=False)
def __init__(self, request, *args, **kwargs):
super(UploadForm, self).__init__(*args, **kwargs)
self.remote_ip = request.META.get('REMOTE_ADDR', None)
self.in_first_cut_off = False
self.cutoff_warning = ""
self.shutdown = False
self.set_cutoff_warnings()
self.group = None
self.parsed_draft = None
def set_cutoff_warnings(self):
from datetime import timedelta
now = datetime.datetime.utcnow()
first_cut_off = Meeting.get_first_cut_off()
second_cut_off = Meeting.get_second_cut_off()
ietf_monday = Meeting.get_ietf_monday()
if now.date() >= (first_cut_off-timedelta(days=settings.CUTOFF_WARNING_DAYS)) and now.date() < first_cut_off:
self.cutoff_warning = ( 'The pre-meeting cut-off date for new documents (i.e., version -00 Internet-Drafts) is %s at %02sh UTC.<br/>' % (first_cut_off, settings.CUTOFF_HOUR) +
'The pre-meeting cut-off date for revisions to existing documents is %s at %02sh UTC.<br/>' % (second_cut_off, settings.CUTOFF_HOUR) )
elif now.date() >= first_cut_off and now.date() < second_cut_off: # We are in the first_cut_off
if now.date() == first_cut_off and now.hour < settings.CUTOFF_HOUR:
self.cutoff_warning = 'The pre-meeting cut-off date for new documents (i.e., version -00 Internet-Drafts) is %s, at %02sh UTC. After that, you will not be able to submit a new document until %s, at %sh UTC' % (first_cut_off, settings.CUTOFF_HOUR, ietf_monday, settings.CUTOFF_HOUR, )
else: # No 00 version allowed
self.cutoff_warning = 'The pre-meeting cut-off date for new documents (i.e., version -00 Internet-Drafts) was %s at %sh UTC. You will not be able to submit a new document until %s, at %sh UTC.<br>You can still submit a version -01 or higher Internet-Draft until %sh UTC, %s' % (first_cut_off, settings.CUTOFF_HOUR, ietf_monday, settings.CUTOFF_HOUR, settings.CUTOFF_HOUR, second_cut_off, )
self.in_first_cut_off = True
elif now.date() >= second_cut_off and now.date() < ietf_monday:
if now.date() == second_cut_off and now.hour < settings.CUTOFF_HOUR: # We are in the first_cut_off yet
self.cutoff_warning = 'The pre-meeting cut-off date for new documents (i.e., version -00 Internet-Drafts) was %s at %02sh UTC. You will not be able to submit a new document until %s, at %02sh UTC.<br>The I-D submission tool will be shut down at %02sh UTC today, and reopened at %02sh UTC on %s' % (first_cut_off, settings.CUTOFF_HOUR, ietf_monday, settings.CUTOFF_HOUR, settings.CUTOFF_HOUR, settings.CUTOFF_HOUR, ietf_monday)
self.in_first_cut_off = True
else: # Completely shut down of the tool
self.cutoff_warning = 'The cut-off time for the I-D submission was %02dh UTC, %s.<br>The I-D submission tool will be reopened at %02dh local time at the IETF meeting location, %s.' % (settings.CUTOFF_HOUR, second_cut_off, settings.CUTOFF_HOUR, ietf_monday)
self.shutdown = True
def clean_file(self, field_name, parser_class):
f = self.cleaned_data[field_name]
if not f:
return f
parsed_info = parser_class(f).critical_parse()
if parsed_info.errors:
raise forms.ValidationError(parsed_info.errors)
return f
def clean_txt(self):
return self.clean_file("txt", PlainParser)
def clean_pdf(self):
return self.clean_file("pdf", PDFParser)
def clean_ps(self):
return self.clean_file("ps", PSParser)
def clean_xml(self):
return self.clean_file("xml", XMLParser)
def clean(self):
if self.shutdown:
raise forms.ValidationError('The tool is shut down')
# sanity check that paths exist (for development servers)
for s in ("IDSUBMIT_STAGING_PATH", "IDSUBMIT_IDNITS_BINARY",
"IDSUBMIT_REPOSITORY_PATH", "INTERNET_DRAFT_ARCHIVE_DIR"):
if not os.path.exists(getattr(settings, s)):
raise forms.ValidationError('%s defined in settings.py does not exist' % s)
if self.cleaned_data.get('txt'):
# try to parse it
txt_file = self.cleaned_data['txt']
txt_file.seek(0)
self.parsed_draft = Draft(txt_file.read(), txt_file.name)
txt_file.seek(0)
if not self.parsed_draft.filename:
raise forms.ValidationError("Draft parser could not extract a valid draft name from the .txt file")
if not self.parsed_draft.get_title():
raise forms.ValidationError("Draft parser could not extract a valid title from the .txt file")
# check group
self.group = self.deduce_group()
# check existing
existing = Submission.objects.filter(name=self.parsed_draft.filename, rev=self.parsed_draft.revision).exclude(state__in=("posted", "cancel"))
if existing:
raise forms.ValidationError(mark_safe('Submission with same name and revision is currently being processed. <a href="%s">Check the status here</a>' % urlreverse("submit_submission_status", kwargs={ 'submission_id': existing[0].pk })))
# cut-off
if self.parsed_draft.revision == '00' and self.in_first_cut_off:
raise forms.ValidationError(mark_safe(self.cutoff_warning))
# check thresholds
today = datetime.date.today()
self.check_submissions_tresholds(
"for the draft %s" % self.parsed_draft.filename,
dict(name=self.parsed_draft.filename, rev=self.parsed_draft.revision, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
)
self.check_submissions_tresholds(
"for the same submitter",
dict(remote_ip=self.remote_ip, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER, settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
)
if self.group:
self.check_submissions_tresholds(
"for the group \"%s\"" % (self.group.acronym),
dict(group=self.group, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP, settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE,
)
self.check_submissions_tresholds(
"across all submitters",
dict(submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
return super(UploadForm, self).clean()
def check_submissions_tresholds(self, which, filter_kwargs, max_amount, max_size):
submissions = Submission.objects.filter(**filter_kwargs)
if len(submissions) > max_amount:
raise forms.ValidationError("Max submissions %s has been reached for today (maximum is %s submissions)." % (which, max_amount))
if sum(s.file_size for s in submissions) > max_size * 1024 * 1024:
raise forms.ValidationError("Max uploaded amount %s has been reached for today (maximum is %s MB)." % (which, max_size))
def deduce_group(self):
"""Figure out group from name or previously submitted draft, returns None if individual."""
name = self.parsed_draft.filename
existing_draft = Document.objects.filter(name=name, type="draft")
if existing_draft:
group = existing_draft[0].group
if group and group.type_id not in ("individ", "area"):
return group
else:
return None
else:
if name.startswith('draft-ietf-') or name.startswith("draft-irtf-"):
components = name.split("-")
if len(components) < 3:
raise forms.ValidationError(u"The draft name \"%s\" is missing a third part, please rename it" % name)
if components[1] == "ietf":
group_type = "wg"
elif components[1] == "irtf":
group_type = "rg"
# first check groups with dashes
for g in Group.objects.filter(acronym__contains="-", type=group_type):
if name.startswith('draft-%s-%s-' % (components[1], g.acronym)):
return g
try:
return Group.objects.get(acronym=components[2], type=group_type)
except Group.DoesNotExist:
raise forms.ValidationError('There is no active group with acronym \'%s\', please rename your draft' % components[2])
elif name.startswith("draft-iab-"):
return Group.objects.get(acronym="iab")
else:
return None
canonical_name = n.name
if canonical_name.startswith("rfc"):
path = os.path.join(settings.RFC_PATH, canonical_name + ".txt")
else:
path = os.path.join(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR,
canonical_name + "-" + doc.rev + ".txt")
if not os.path.exists(path):
say("Skipping %s, no txt file found at %s" % (doc.name, path))
continue
with open(path, 'rb') as f:
say("\nProcessing %s" % doc.name)
sys.stdout.flush()
d = Draft(unicode(f.read()), path)
updated = False
updates = {}
if args.words:
words = d.get_wordcount()
if words != doc.words:
updates["words"] = words
if args.formlang:
langs = d.get_formal_languages()
new_formal_languages = set(formal_language_dict[l] for l in langs)
old_formal_languages = set(doc.formal_languages.all())
if not Document.objects.filter(name=name).exists():
paths = list(
Path(settings.INTERNET_DRAFT_PATH).glob('%s-??.txt' % name))
paths.sort()
doc = None
for p in paths:
n = str(p).split('/')[-1].split('-')
rev = n[-1][:2]
with open(str(p)) as txt_file:
raw = txt_file.read()
try:
text = raw.decode('utf8')
except UnicodeDecodeError:
text = raw.decode('latin1')
try:
draft = Draft(text, txt_file.name, name_from_source=True)
except Exception as e:
print name, rev, "Can't parse", p, ":", e
continue
if draft.errors and draft.errors.keys() != [
'draftname',
]:
print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(
p.stat().st_mtime), draft.errors, draft.get_title().encode(
'utf8')
else:
time = datetime.datetime.fromtimestamp(p.stat().st_mtime)
if not doc:
doc = Document.objects.create(
name=name,
time=time,
def get_draft_meta(form):
authors = []
file_name = {}
abstract = None
file_size = None
for ext in form.fields.keys():
if not ext in form.formats:
continue
f = form.cleaned_data[ext]
if not f:
continue
name = os.path.join(settings.IDSUBMIT_STAGING_PATH,
'%s-%s.%s' % (form.filename, form.revision, ext))
file_name[ext] = name
with open(name, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
if form.cleaned_data['xml']:
if not ('txt' in form.cleaned_data and form.cleaned_data['txt']):
file_name['txt'] = os.path.join(
settings.IDSUBMIT_STAGING_PATH,
'%s-%s.txt' % (form.filename, form.revision))
try:
pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree,
quiet=True)
pagedwriter.write(file_name['txt'])
except Exception as e:
raise ValidationError("Error from xml2rfc: %s" % e)
file_size = os.stat(file_name['txt']).st_size
# Some meta-information, such as the page-count, can only
# be retrieved from the generated text file. Provide a
# parsed draft object to get at that kind of information.
with open(file_name['txt']) as txt_file:
form.parsed_draft = Draft(txt_file.read().decode('utf8'),
txt_file.name)
else:
file_size = form.cleaned_data['txt'].size
if form.authors:
authors = form.authors
else:
# If we don't have an xml file, try to extract the
# relevant information from the text file
for author in form.parsed_draft.get_author_list():
full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author
name = full_name.replace("\n", "").replace("\r", "").replace(
"<", "").replace(">", "").strip()
if email:
try:
validate_email(email)
except ValidationError:
email = ""
def turn_into_unicode(s):
if s is None:
return u""
if isinstance(s, unicode):
return s
else:
try:
return s.decode("utf-8")
except UnicodeDecodeError:
try:
return s.decode("latin-1")
except UnicodeDecodeError:
return ""
name = turn_into_unicode(name)
email = turn_into_unicode(email)
company = turn_into_unicode(company)
authors.append({
"name": name,
"email": email,
"affiliation": company,
"country": country
})
if form.abstract:
abstract = form.abstract
else:
abstract = form.parsed_draft.get_abstract()
return authors, abstract, file_name, file_size