def _save_template(request, assets, context):
"""Save a template.
Args:
request:
assets:
context:
Returns:
"""
# get the schema name
name = request.POST['name']
# get the file from the form
xsd_file = request.FILES['upload_file']
# read the content of the file
xsd_data = read_xsd_file(xsd_file)
try:
template = Template(filename=xsd_file.name, content=xsd_data)
template_version_manager = TemplateVersionManager(title=name)
template_version_manager_api.insert(template_version_manager, template)
return HttpResponseRedirect(reverse("admin:core_main_app_templates"))
except exceptions.XSDError as xsd_error:
return handle_xsd_errors(request, assets, context, xsd_error, xsd_data,
xsd_file.name)
except exceptions.NotUniqueError:
context['errors'] = html_escape(
"A template with the same name already exists. Please choose another name."
)
return _upload_template_response(request, assets, context)
except Exception as e:
context['errors'] = html_escape(str(e))
return _upload_template_response(request, assets, context)
def _save_xslt(self, request):
"""Saves an XSLT.
Args:
request: Request.
"""
try:
# get the XSLT name
name = request.POST['name']
# get the file from the form
xsd_file = request.FILES['upload_file']
# read the content of the file
xsd_data = read_xsd_file(xsd_file)
xslt = XslTransformation(name=name,
filename=xsd_file.name,
content=xsd_data)
xslt_transformation_api.upsert(xslt)
return HttpResponseRedirect(reverse("admin:core_main_app_xslt"))
except exceptions.NotUniqueError:
self.context.update(
{'errors': html_escape("This name already exists.")})
return admin_render(request,
'core_main_app/admin/xslt/upload.html',
context=self.context)
except Exception, e:
self.context.update({'errors': html_escape(e.message)})
return admin_render(request,
'core_main_app/admin/xslt/upload.html',
context=self.context)
def join_escaped_html(self, splitted_data, pos_left, pos_right):
lines = splitted_data[pos_left[0]-1:pos_right[0]]
if len(lines) == 1:
return html_escape(lines[0][pos_left[1]:pos_right[1]+1])
else:
lines[0] = lines[0][pos_left[1]:]
lines[-1] = lines[-1][:pos_right[1]+1]
return html_escape("\n".join(lines))
def join_escaped_html(self, splitted_data, pos_left, pos_right):
lines = splitted_data[pos_left[0] - 1:pos_right[0]]
if len(lines) == 1:
return html_escape(lines[0][pos_left[1]:pos_right[1] + 1])
else:
lines[0] = lines[0][pos_left[1]:]
lines[-1] = lines[-1][:pos_right[1] + 1]
return html_escape("\n".join(lines))
def save( self, *args, **kwargs ): if self.id: self.default_price = self.get_default_price() self.short_configuration_str = self.short_configuration() self.description_html = publish_parts( html_escape( self.description ), writer_name="html4css1" )["fragment"] self.description_html_en = publish_parts( html_escape( self.description_en ), writer_name="html4css1" )["fragment"] self.last_modified = datetime.datetime.now() super( ComputerModel, self ).save(*args, **kwargs)
def upload_query_ontology(request):
"""Upload ontology.
Args:
request:
Returns:
"""
assets = {
"js": [{"path": "core_main_app/common/js/backtoprevious.js", "is_raw": True}]
}
context = {
"object_name": "OWL files",
"url": reverse("admin:core_explore_tree_app_upload"),
"redirect_url": reverse("admin:core_explore_tree_app_query_ontology"),
}
# method is POST
if request.method == "POST":
form = UploadQueryOntologyForm(request.POST, request.FILES, request=request)
context["upload_form"] = form
if form.is_valid():
try:
# save the query ontology
_save_query_ontology(request, context)
# redirect to the list of query ontology
return HttpResponseRedirect(
reverse("admin:core_explore_tree_app_query_ontology")
)
except exceptions.NotUniqueError:
context["errors"] = html_escape(
"An Ontology with the same name already exists. "
"Please choose another name."
)
except Exception as e:
context["errors"] = html_escape(str(e))
# method is GET
else:
# render the form to upload a query ontology
context["upload_form"] = UploadQueryOntologyForm(request=request)
# render the upload page
return admin_render(
request,
"core_explore_tree_app/admin/query_ontology/upload.html",
assets=assets,
context=context,
)
def _save_template_xslt(self, request):
"""Save a template xslt rendering.
Args:
request: Request.
"""
try:
# Get the list xslt instance
try:
list_xslt = xslt_transformation_api.get_by_id(request.POST.get('list_xslt'))
except (Exception, exceptions.DoesNotExist):
list_xslt = None
# Get the detail xslt instance
try:
detail_xslt = xslt_transformation_api.get_by_id(request.POST.get('detail_xslt'))
except (Exception, exceptions.DoesNotExist):
detail_xslt = None
template_xsl_rendering_api.add_or_delete(template_xsl_rendering_id=request.POST.get('id'),
template_id=request.POST.get('template'),
list_xslt=list_xslt, detail_xslt=detail_xslt)
template = template_api.get(request.POST.get('template'))
# Get template information (version)
version_manager = version_manager_api.get_from_version(template)
return HttpResponseRedirect(reverse(self.save_redirect, args=[version_manager.id]))
except Exception, e:
self.context.update({'errors': html_escape(e.message)})
return self.rendering(request, self.template_name, context=self.context)
def _save_template_version(request, assets, context, template_version_manager):
"""Save a template version.
Args:
request:
assets:
context:
template_version_manager:
Returns:
"""
# get the file from the form
xsd_file = request.FILES['xsd_file']
# read the content of the file
xsd_data = read_xsd_file(xsd_file)
try:
template = Template(filename=xsd_file.name, content=xsd_data)
template_version_manager_api.insert(template_version_manager, template)
return HttpResponseRedirect(
reverse("admin:core_main_app_manage_template_versions",
kwargs={
'version_manager_id': str(template_version_manager.id)
}))
except exceptions.XSDError as xsd_error:
return handle_xsd_errors(request, assets, context, xsd_error, xsd_data,
xsd_file.name)
except Exception as e:
context['errors'] = html_escape(str(e))
return _upload_template_response(request, assets, context)
def handle_xsd_errors(request, assets, context, xsd_error, xsd_content,
filename):
"""Handle XSD errors. Builds dependency resolver if needed.
Args:
request:
assets:
context:
xsd_error:
xsd_content:
filename:
Returns:
"""
imports, includes = get_imports_and_includes(xsd_content)
# a problem with includes/imports has been detected
if len(includes) > 0 or len(imports) > 0:
# build dependency resolver
context['dependency_resolver'] = get_dependency_resolver_html(
imports, includes, xsd_content, filename)
return _upload_template_response(request, assets, context)
else:
context['errors'] = html_escape(xsd_error.message)
return _upload_template_response(request, assets, context)
def _save_custom_resources(self, request, template_id):
"""Saves an XSLT.
Args:
request: Request.
template_id
"""
try:
# get the template
template = template_api.get(template_id, request=request)
# get the file from the form
upload_file = request.FILES["json_file"].read().decode("utf-8")
data = json.loads(upload_file)
custom_resource_api.replace_custom_resources_by_template(
template, data)
return HttpResponseRedirect(
reverse("admin:core_main_registry_app_custom_registry"))
except Exception as e:
self.context.update({"errors": html_escape(str(e))})
return admin_render(request,
self.template_name,
context=self.context)
def _save_type(request, assets, context):
"""Save a type.
Args:
request:
assets:
context:
Returns:
"""
try:
# get the schema name
name = request.POST["name"]
# get the file from the form
xsd_file = request.FILES["xsd_file"]
# read the content of the file
xsd_data = read_xsd_file(xsd_file)
# get the buckets
buckets = request.POST.getlist("buckets")
type_object = Type(filename=xsd_file.name, content=xsd_data)
type_version_manager = TypeVersionManager(title=name)
type_version_manager_api.insert(type_version_manager,
type_object,
request=request,
list_bucket_ids=buckets)
return HttpResponseRedirect(reverse("admin:core_composer_app_types"))
except exceptions.XSDError as xsd_error:
return _handle_xsd_errors(request, assets, context, xsd_error,
xsd_data, xsd_file.name)
except Exception as e:
context["errors"] = html_escape(str(e))
return _upload_type_response(request, assets, context)
def _escape(self):
# eliminamos los chars especiales
# https://lucene.apache.org/core/2_9_4/queryparsersyntax.html#Escaping%20Special%20Characters
map(self._replace, ("+","-","&&","||","!","(",")","{","}","[","]","^","\"","~","*","?",":","\\", "/"))
return html_escape(self.q)
def _save_type_version(request, assets, context, type_version_manager):
"""Save a type version.
Args:
request:
assets:
context:
type_version_manager:
Returns:
"""
# get the file from the form
xsd_file = request.FILES["xsd_file"]
# read the content of the file
xsd_data = read_xsd_file(xsd_file)
try:
type_object = Type(filename=xsd_file.name, content=xsd_data)
type_version_manager_api.insert(type_version_manager,
type_object,
request=request)
return HttpResponseRedirect(
reverse(
"admin:core_composer_app_manage_type_versions",
kwargs={"version_manager_id": str(type_version_manager.id)},
))
except exceptions.XSDError as xsd_error:
return _handle_xsd_errors(request, assets, context, xsd_error,
xsd_data, xsd_file.name)
except Exception as e:
context["errors"] = html_escape(str(e))
return _upload_type_response(request, assets, context)
def _save_xslt(self, request):
"""Save an XSLT.
Args:
request: Request.
"""
try:
# get the XSLT name
name = request.POST['name']
# get the file from the form
xsd_file = request.FILES['upload_file']
# read the content of the file
xsd_data = read_xsd_file(xsd_file)
xslt = XslTransformation(name=name,
filename=xsd_file.name,
content=xsd_data)
xslt_transformation_api.upsert(xslt)
return HttpResponseRedirect(reverse("core_main_app_xslt"))
except Exception as e:
self.context.update({'errors': html_escape(str(e))})
return render(request,
'core_main_app/common/xslt/upload.html',
context=self.context)
def _get_dependency_resolver_html(imports, includes, xsd_data, filename):
"""Return HTML for dependency resolver form.
Args:
imports:
includes:
xsd_data:
filename:
Returns:
"""
# build the list of dependencies
current_types = type_version_manager_api.get_global_version_managers()
list_dependencies_template = loader.get_template(
'core_main_app/admin/list_dependencies.html')
context = {
'templates':
[template for template in current_types if not template.is_disabled]
}
list_dependencies_html = list_dependencies_template.render(context)
# build the dependency resolver form
dependency_resolver_template = loader.get_template(
'core_main_app/admin/dependency_resolver.html')
context = {
'imports': imports,
'includes': includes,
'xsd_content': html_escape(xsd_data),
'filename': filename,
'dependencies': list_dependencies_html,
}
return dependency_resolver_template.render(context)
def _escape(self):
"""metodo que devuelve el q limpio de caracteres especiales"""
# eliminamos los chars especiales
# https://lucene.apache.org/core/2_9_4/queryparsersyntax.html#Escaping%20Special%20Characters
map(self._replace, ("+","-","&&","||","!","(",")","{","}","[","]","^","\"","~","*","?",":","\\", "/"))
return html_escape(self.cleaned_data['q'])
def _escape(self):
"""metodo que devuelve el q limpio de caracteres especiales"""
# eliminamos los chars especiales
# https://lucene.apache.org/core/2_9_4/queryparsersyntax.html#Escaping%20Special%20Characters
map(self._replace, ("+", "-", "&&", "||", "!", "(", ")", "{", "}", "[",
"]", "^", "\"", "~", "*", "?", ":", "\\", "/"))
return html_escape(self.cleaned_data['q'])
def story_html_to_text(content, clean=True):
"""
>>> content = '''<html><body>
... <pre>hello world</pre>
...
...
... <p>happy day</p>
... </body></html>
... '''
>>> print(story_html_to_text(content))
happy day
>>> print(story_html_to_text(content, clean=False))
hello world
happy day
>>> content = '<![CDATA[hello world]]>'
>>> print(story_html_to_text(content))
hello world
>>> print(story_html_to_text('<pre><code>hi</code></pre>'))
<BLANKLINE>
>>> content = '''
... <?xml version="1.0" encoding="utf-8"?>
... <?xml-stylesheet type="text/xsl" href="/res/preview.xsl"?>
... <p>中文传媒精选</p>
... '''
>>> print(story_html_to_text(content))
中文传媒精选
>>> story_html_to_text('') == ''
True
>>> # lxml can not parse below content, we handled the exception
>>> content = "<?phpob_start();echo file_get_contents($_GET['pdf_url']);ob_flush();?>"
>>> assert story_html_to_text(content)
"""
if (not content) or (not content.strip()):
return ""
try:
if clean:
# https://bugs.launchpad.net/lxml/+bug/1851029
# The html cleaner raise AssertionError when both
# root tag and child tag in kill_tags set.
if content.startswith('<pre'):
content = '<div>' + content + '</div>'
content = lxml_call(lxml_text_html_cleaner.clean_html,
content).strip()
if not content:
return ""
r = lxml_call(lxml.html.fromstring, content, parser=lxml_html_parser)
content = r.text_content().strip()
except LXMLError:
try:
content = lxml_call(soupparser.fromstring,
content).text_content().strip()
except LXMLError as ex:
LOG.info(f'lxml unable to parse content: {ex} content={content!r}',
exc_info=ex)
content = html_escape(content)
return RE_BLANK_LINE.sub('\n', content)
def upload_query_ontology(request):
""" Upload ontology.
Args:
request:
Returns:
"""
assets = {
"js": [{
"path": 'core_main_app/common/js/backtoprevious.js',
"is_raw": True
}]
}
context = {
'object_name': 'OWL files',
'url': reverse("admin:core_explore_tree_app_upload"),
'redirect_url': reverse("admin:core_explore_tree_app_query_ontology")
}
# method is POST
if request.method == 'POST':
form = UploadQueryOntologyForm(request.POST, request.FILES)
context['upload_form'] = form
if form.is_valid():
try:
# save the query ontology
_save_query_ontology(request, context)
# redirect to the list of query ontology
return HttpResponseRedirect(
reverse("admin:core_explore_tree_app_query_ontology"))
except exceptions.NotUniqueError:
context['errors'] = html_escape(
"An Ontology with the same name already exists. "
"Please choose another name.")
except Exception, e:
context['errors'] = html_escape(e.message)
def clean_user_agent(self):
"""
Get the user agent, either the browser user agent or the configured user agent
if any
"""
try:
uaBrowser = self.headers.get('HTTP_USER_AGENT', '').decode('utf-8')
except:
# For old application sending user agent in latin-1 ..
uaBrowser = self.headers.get('HTTP_USER_AGENT', '').decode('latin-1')
uaBrowser = uaBrowser.strip().replace(' ', '')
# Select name from the name cookie if any
uaCookie = self.cookies.get(TRIBUNE_MESSAGES_UA_COOKIE_NAME, False)
if uaCookie and len(uaCookie)>2:
from base64 import b64decode
return html_escape( b64decode(uaCookie) )
# Séléction de l'ua donnée dans le client
else:
if len(uaBrowser) < TRIBUNE_MESSAGES_UA_LENGTH_MIN:
return "coward"
return html_escape( uaBrowser )
def clean_user_agent(self):
"""
Get the user agent, either the browser user agent or the configured user agent
if any
"""
try:
uaBrowser = self.headers.get('HTTP_USER_AGENT', '').decode('utf-8')
except:
# For old application sending user agent in latin-1 ..
uaBrowser = self.headers.get('HTTP_USER_AGENT', '').decode('latin-1')
uaBrowser = uaBrowser.strip().replace(' ', '')
# Select name from the name cookie if any
uaCookie = self.cookies.get(TRIBUNE_MESSAGES_UA_COOKIE_NAME, False)
if uaCookie and len(uaCookie)>2:
from base64 import b64decode
return html_escape( b64decode(uaCookie) )
# Séléction de l'ua donnée dans le client
else:
if len(uaBrowser) < TRIBUNE_MESSAGES_UA_LENGTH_MIN:
return "coward"
return html_escape( uaBrowser )
def story_html_clean(content, loose=False):
"""
>>> content = '''<html><head><style></style></head><body>
... <pre stype="xxx">
...
... hello world</pre>
... <p><b>happy</b> day<br>你好<i>世界</i></p>
... </body></html>
... '''
>>> print(story_html_clean(content))
<div>
<pre>
<BLANKLINE>
hello world</pre>
<p><b>happy</b> day<br>你好<i>世界</i></p>
</div>
>>> content = '''
... <?xml version="1.0" encoding="utf-8"?>
... <?xml-stylesheet type="text/xsl" href="/res/preview.xsl"?>
... <p>中文传媒精选</p>
... '''
>>> print(story_html_clean(content))
<p>中文传媒精选</p>
>>> # lxml can not parse below content, we handled the exception
>>> content = '<!-- build time:Mon Mar 16 2020 19:23:52 GMT+0800 (GMT+08:00) --><!-- rebuild by neat -->'
>>> assert story_html_clean(content)
>>> # loose cleaner allow iframe, not allow embed flash
>>> content = '<iframe src="https://example.com/123" width="650" height="477" border="0"></iframe>'
>>> story_html_clean(content)
'<div></div>'
>>> 'iframe' in story_html_clean(content, loose=True)
True
>>> content = '<embed src="https://example.com/movie.mp4">'
>>> story_html_clean(content, loose=True)
'<div></div>'
>>> content = '<svg height="16" width="16" class="octicon octicon-search"></svg>'
>>> story_html_clean(content) == content
True
"""
if (not content) or (not content.strip()):
return ""
cleaner = lxml_story_html_loose_cleaner if loose else lxml_story_html_cleaner
try:
content = lxml_call(cleaner.clean_html, content).strip()
except LXMLError as ex:
LOG.info(f'lxml unable to parse content: {ex} content={content!r}',
exc_info=ex)
content = html_escape(content)
if not content:
return ""
return content
def test_send_with_incorrect_application_token(self):
with mock.patch('jepostule.auth.utils.make_application_token',
return_value='apptoken'):
response = self.client.post(
reverse('embed:candidater'),
data=self.form_data(token='invalid apptoken'),
)
self.assertEqual(200, response.status_code)
self.assertIn(html_escape("Jeton d'authentification invalide"),
response.content.decode())
application.send_application_to_employer.consume()
application.send_confirmation_to_candidate.consume()
self.assertEqual([], mail.outbox)
def _save_template_xslt(self, request):
"""Save a template xslt rendering.
Args:
request: Request.
"""
try:
# Get the list xslt instance
try:
list_xslt = xslt_transformation_api.get_by_id(
request.POST.get("list_xslt"))
except (Exception, exceptions.DoesNotExist):
list_xslt = None
# Get the list detail xslt instance
try:
list_detail_xslt = xslt_transformation_api.get_by_id_list(
request.POST.getlist("list_detail_xslt"))
except (Exception, exceptions.DoesNotExist):
list_detail_xslt = None
# Get the default detail xslt instance
try:
default_detail_xslt = xslt_transformation_api.get_by_id(
request.POST.get("default_detail_xslt"))
except (Exception, exceptions.DoesNotExist):
default_detail_xslt = None
template_xsl_rendering_api.add_or_delete(
template_xsl_rendering_id=request.POST.get("id"),
template_id=request.POST.get("template"),
list_xslt=list_xslt,
default_detail_xslt=default_detail_xslt,
list_detail_xslt=list_detail_xslt,
)
template = template_api.get(request.POST.get("template"),
request=request)
# Get template information (version)
version_manager = version_manager_api.get_from_version(
template, request=request)
return HttpResponseRedirect(
reverse(self.save_redirect, args=[version_manager.id]))
except Exception as e:
self.context.update({"errors": html_escape(str(e))})
return self.rendering(request,
self.template_name,
context=self.context)
def render_value_in_context(value, context):
"""Render value in context.
Converts any value to a string to become part of a rendered template.
This means escaping, if required, and conversion to a unicode object.
If value is a string, it is expected to have already been translated.
"""
value = template_localtime(value, use_tz=context.use_tz)
value = localize(value, use_l10n=context.use_l10n)
value = force_text(value)
if ((context.autoescape and not isinstance(value, SafeData))
or isinstance(value, EscapeData)):
return html_escape(value)
else:
return value
def render_value_in_context(value, context):
"""Render value in context.
Converts any value to a string to become part of a rendered template.
This means escaping, if required, and conversion to a unicode object.
If value is a string, it is expected to have already been translated.
"""
value = template_localtime(value, use_tz=context.use_tz)
value = localize(value, use_l10n=context.use_l10n)
value = force_text(value)
if ((context.autoescape and not isinstance(value, SafeData)) or
isinstance(value, EscapeData)):
return html_escape(value)
else:
return value
def test_answer_interview_post(self):
response = self.client.post(
self.job_application.get_answer_url(models.Answer.Types.INTERVIEW),
data=interview_form_data(),
)
self.assertEqual(200, response.status_code)
self.assertIn(
html_escape(forms.InterviewForm.success_message),
response.content.decode()
)
self.assertEqual('Jessica Lange', self.job_application.answer.answerinterview.employer_name)
self.assertEqual(
timezone.make_aware(datetime(2051, 12, 31, 8)).astimezone(timezone.utc),
self.job_application.answer.answerinterview.datetime
)
def rssFeed(request):
log.debug( 'starting rssFeed()' )
log.debug( 'request, ```%s```' % pprint.pformat(request.__dict__) )
from django.utils.html import escape as html_escape
def remove_html_tags(data):
p = re.compile(r'<.*?>')
p = p.sub('', data)
return p.replace('"', ' ')
context = RequestContext(request)
log.debug( 'type(context), `%s`' % type(context) )
log.debug( 'context.__dict__ before update, ```%s```' % pprint.pformat(context.__dict__) )
context.update(get_search_results(request))
log.debug( 'context.__dict__ after update, ```%s```' % pprint.pformat(context.__dict__) )
context['ILS'] = settings.ILS
results = context['response']['docs']
limits_param = request.GET.get('limits', '')
limits, fq_params = pull_limits(limits_param)
query = request.GET.get('q', '')
log.debug( 'query, ```%s```' % query )
full_query_str = get_full_query_str(query, limits)
log.debug( 'full_query_str, ```%s```' % full_query_str )
feed = feedgenerator.Rss201rev2Feed(title='BUL new books in %s' % remove_html_tags(full_query_str),
link=settings.CATALOG_URL,
description='BUL new books %s' % remove_html_tags(full_query_str),
language=u"en")
for result in results:
if result.has_key('discipline'):
summary = "%s. " % ", ".join(sorted(result['discipline']))
else:
summary = ""
if result.has_key('summary'):
if len(result['summary']) > 0:
summary += "%s." % result['summary'][0]
feed.add_item(title=result['title'],
link=result['record_url'],
unique_id=result['record_url'],
description=html_escape(summary)
#pubdate=result['accession_date']
)
# response = HttpResponse(mimetype='application/xml')
response = HttpResponse(content_type='application/xml')
feed.write(response, 'utf-8')
return response
def _save_template_version(request, assets, context, template_version_manager):
"""Save a template version.
Args:
request:
assets:
context:
template_version_manager:
Returns:
"""
try:
# get the file from the form
xsd_file = request.FILES["xsd_file"]
# read the content of the file
xsd_data = read_xsd_file(xsd_file)
template = Template(filename=xsd_file.name, content=xsd_data)
template_version_manager_api.insert(template_version_manager,
template,
request=request)
# create the fragment url with all the version of the template (minus the new template)
version_manager_string = ""
for version in template_version_manager.versions:
if version != str(template.id):
current_version_string = (version if version_manager_string
== "" else f",{version}")
version_manager_string += current_version_string
# add the fragment data to the url
fragment = f"#from={version_manager_string}&to={template.id}"
return HttpResponseRedirect(
reverse("admin:core_main_app_data_migration") + fragment)
except exceptions.XSDError as xsd_error:
return handle_xsd_errors(request, assets, context, xsd_error, xsd_data,
xsd_file.name)
except Exception as e:
context["errors"] = html_escape(str(e))
return _upload_template_response(request, assets, context)
def story_html_clean(content):
"""
>>> content = '''<html><head><style></style></head><body>
... <pre stype="xxx">
...
... hello world</pre>
... <p>happy day</p>
... </body></html>
... '''
>>> print(story_html_clean(content))
<div>
<pre>
<BLANKLINE>
hello world</pre>
<p>happy day</p>
</div>
>>> content = '''
... <?xml version="1.0" encoding="utf-8"?>
... <?xml-stylesheet type="text/xsl" href="/res/preview.xsl"?>
... <p>中文传媒精选</p>
... '''
>>> print(story_html_clean(content))
<p>中文传媒精选</p>
>>> # lxml can not parse below content, we handled the exception
>>> content = '<!-- build time:Mon Mar 16 2020 19:23:52 GMT+0800 (GMT+08:00) --><!-- rebuild by neat -->'
>>> assert story_html_clean(content)
"""
if (not content) or (not content.strip()):
return ""
try:
content = lxml_call(lxml_story_html_cleaner.clean_html,
content).strip()
except LXMLError as ex:
LOG.info(f'lxml unable to parse content: {ex} content={content!r}',
exc_info=ex)
content = html_escape(content)
if not content:
return ""
return content
def get_dependency_resolver_html(imports, includes, xsd_data, filename,
request):
"""Return HTML for dependency resolver form.
Args:
imports:
includes:
xsd_data:
filename:
request:
Returns:
"""
# build the list of dependencies
current_templates = template_version_manager_api.get_global_version_managers(
request=request, _cls=False)
list_dependencies_template = loader.get_template(
"core_main_app/admin/list_dependencies.html")
context = {
"templates": [
template for template in current_templates
if not template.is_disabled
],
}
list_dependencies_html = list_dependencies_template.render(context)
# build the dependency resolver form
dependency_resolver_template = loader.get_template(
"core_main_app/admin/dependency_resolver.html")
context = {
"imports": imports,
"includes": includes,
"xsd_content": html_escape(xsd_data),
"filename": filename,
"dependencies": list_dependencies_html,
}
return dependency_resolver_template.render(context)
def _handle_xsd_errors(request, assets, context, xsd_error, xsd_content,
filename):
"""Handle XSD errors. Builds dependency resolver if needed.
Args:
request:
context:
xsd_error:
xsd_content:
filename:
Returns:
"""
imports, includes = get_imports_and_includes(xsd_content)
# a problem with includes/imports has been detected
if len(includes) > 0 or len(imports) > 0:
# build dependency resolver
context["dependency_resolver"] = _get_dependency_resolver_html(
imports, includes, xsd_content, filename, request=request)
return _upload_type_response(request, assets, context)
else:
context["errors"] = html_escape(str(xsd_error))
return _upload_type_response(request, assets, context)
def parseDocObjectsToStrings(records, obj_type):
"""
called by parseDocumentsForW2ui and get_table_data to convert some of
the objects in the record dictionaries into strings.
For example converts the sources into their names instead of returning the
entire object
"""
for doc in records:
for key, value in doc.items():
# all dates should look the same
if isinstance(value, datetime.datetime):
doc[key] = datetime.datetime.strftime(value,
"%Y-%m-%d %H:%M:%S")
if key == "_id" or key == "id":
doc["recid"] = str(value)
doc["details"] = "<a href='"+getHREFLink(doc, obj_type)+"'>"\
"<div class='icon-container'>"\
"<span class='ui-icon ui-icon-document'></span>"\
"</div>"\
"</a>"
elif key == "password_reset":
doc['password_reset'] = None
elif key == "exploit":
exploits = []
for ex in value:
exploits.append(ex['cve'])
doc[key] = "|||".join(exploits)
elif key == "campaign":
camps = []
for campdict in value:
camps.append(campdict['name'])
doc[key] = "|||".join(camps)
elif key == "source":
srcs = []
for srcdict in doc[key]:
srcs.append(srcdict['name'])
doc[key] = "|||".join(srcs)
elif key == "tags":
tags = []
for tag in doc[key]:
tags.append(tag)
doc[key] = "|||".join(tags)
elif key == "is_active":
if value:
doc[key] = "True"
else:
doc[key] = "False"
elif key == "tickets":
tickets = []
for ticketdict in value:
tickets.append(ticketdict['ticket_number'])
doc[key] = "|||".join(tickets)
elif key == "datatype":
doc[key] = value.keys()[0]
elif key == "to":
doc[key] = len(value)
elif key == "thumb":
doc['url'] = reverse("crits.screenshots.views.render_screenshot",
args=(unicode(doc["_id"]),))
elif key=="results" and obj_type == "AnalysisResult":
doc[key] = len(value)
elif isinstance(value, list):
if value:
for item in value:
if not isinstance(item, basestring):
break
else:
doc[key] = ",".join(value)
else:
doc[key] = ""
doc[key] = html_escape(doc[key])
value = doc[key].strip()
if isinstance(value, unicode) or isinstance(value, str):
val = ' '.join(value.split())
val = val.replace('"',"'")
doc[key] = val
return records
def escape(value):
from django.utils.html import escape as html_escape, strip_tags
if value:
value = html_escape(strip_tags(value))
return value
def audiofy(match):
url = match.group(0)
url = html_escape(url)
html = '<audio controls><source src="{url}"><a href="{url}">{url}</a></audio>'.format(url=url)
return self.markdown.htmlStash.store(html, safe=True)
def parseDocObjectsToStrings(records, obj_type):
"""
called by parseDocumentsForW2ui and get_table_data to convert some of
the objects in the record dictionaries into strings.
For example converts the sources into their names instead of returning the
entire object
"""
for doc in records:
for key, value in doc.items():
# all dates should look the same
if isinstance(value, datetime.datetime):
doc[key] = datetime.datetime.strftime(value,
"%Y-%m-%d %H:%M:%S")
if key == "_id" or key == "id":
doc["recid"] = str(value)
doc["details"] = "<a href='"+getHREFLink(doc, obj_type)+"'>"\
"<div class='icon-container'>"\
"<span class='ui-icon ui-icon-document'></span>"\
"</div>"\
"</a>"
elif key == "password_reset":
doc['password_reset'] = None
elif key == "campaign":
camps = []
for campdict in value:
camps.append(campdict['name'])
doc[key] = "|||".join(camps)
elif key == "source":
srcs = []
for srcdict in doc[key]:
srcs.append(srcdict['name'])
doc[key] = "|||".join(srcs)
elif key == "tags":
tags = []
for tag in doc[key]:
tags.append(tag)
doc[key] = "|||".join(tags)
elif key == "is_active":
if value:
doc[key] = "True"
else:
doc[key] = "False"
elif key == "tickets":
tickets = []
for ticketdict in value:
tickets.append(ticketdict['ticket_number'])
doc[key] = "|||".join(tickets)
elif key == "datatype":
doc[key] = value.keys()[0]
elif key == "to":
doc[key] = len(value)
elif key == "thumb":
doc['url'] = reverse(
"crits.screenshots.views.render_screenshot",
args=(unicode(doc["_id"]), ))
elif key == "results" and obj_type == "AnalysisResult":
doc[key] = len(value)
elif isinstance(value, list):
if value:
for item in value:
if not isinstance(item, basestring):
break
else:
doc[key] = ",".join(value)
else:
doc[key] = ""
doc[key] = html_escape(doc[key])
value = doc[key].strip()
if isinstance(value, unicode) or isinstance(value, str):
val = ' '.join(value.split())
val = val.replace('"', "'")
doc[key] = val
return records
def _cast_ballot(self, election_id, username, password, need_login=True, check_user_logged_in=False):
"""
check_user_logged_in looks for the "you're already logged" message
"""
# vote by preparing a ballot via the server-side encryption
response = self.app.post("/helios/elections/%s/encrypt-ballot" % election_id, {
'answers_json': utils.to_json([[1]])})
self.assertContains(response, "answers")
# parse it as an encrypted vote with randomness, and make sure randomness is there
the_ballot = utils.from_json(response.testbody)
assert the_ballot['answers'][0].has_key('randomness'), "no randomness"
assert len(the_ballot['answers'][0]['randomness']) == 2, "not enough randomness"
# parse it as an encrypted vote, and re-serialize it
ballot = datatypes.LDObject.fromDict(utils.from_json(response.testbody), type_hint='legacy/EncryptedVote')
encrypted_vote = ballot.serialize()
# cast the ballot
response = self.app.post("/helios/elections/%s/cast" % election_id, {
'encrypted_vote': encrypted_vote})
self.assertRedirects(response, "%s/helios/elections/%s/cast_confirm" % (settings.GET_SECURE_URL_HOST(request), election_id))
cast_confirm_page = response.follow()
if need_login:
if check_user_logged_in:
self.assertContains(cast_confirm_page, "You are logged in as")
self.assertContains(cast_confirm_page, "requires election-specific credentials")
# set the form
login_form = cast_confirm_page.form
login_form['voter_id'] = username
login_form['password'] = password
# we skip that intermediary page now
# cast_confirm_page = login_form.submit()
response = login_form.submit()
# self.assertRedirects(cast_confirm_page, "/helios/elections/%s/cast_confirm" % election_id)
# cast_confirm_page = cast_confirm_page.follow()
else:
# here we should be at the cast-confirm page and logged in
self.assertContains(cast_confirm_page, "VOTE con esta papeleta")
# confirm the vote, now with the actual form
cast_form = cast_confirm_page.form
if 'status_update' in cast_form.fields.keys():
cast_form['status_update'] = False
response = cast_form.submit()
self.assertRedirects(response, "%s/helios/elections/%s/cast_done" % (settings.URL_HOST, election_id))
# at this point an email should have gone out to the user
# at position num_messages after, since that was the len() before we cast this ballot
email_message = mail.outbox[len(mail.outbox) - 1]
url = re.search('http://[^/]+(/[^ \n]*)', email_message.body).group(1)
# check that we can get at that URL
if not need_login:
# confusing piece: if need_login is True, that means it was a public election
# that required login before casting a ballot.
# so if need_login is False, it was a private election, and we do need to re-login here
# we need to re-login if it's a private election, because all data, including ballots
# is otherwise private
login_page = self.app.get("/helios/elections/%s/password_voter_login" % election_id)
# if we redirected, that's because we can see the page, I think
if login_page.status_int != 302:
login_form = login_page.form
# try with extra spaces
login_form['voter_id'] = ' ' + username + ' '
login_form['password'] = '******' + password + ' '
login_form.submit()
response = self.app.get(url)
self.assertContains(response, ballot.hash)
self.assertContains(response, html_escape(encrypted_vote))
# if we request the redirect to cast_done, the voter should be logged out, but not the user
response = self.app.get("/helios/elections/%s/cast_done" % election_id)
# read the content of the file
xsd_data = read_xsd_file(xsd_file)
# get the buckets
buckets = request.POST.getlist('buckets')
try:
type_object = Type(filename=xsd_file.name, content=xsd_data)
type_version_manager = TypeVersionManager(title=name)
type_version_manager_api.insert(type_version_manager, type_object,
buckets)
return HttpResponseRedirect(reverse("admin:core_composer_app_types"))
except exceptions.XSDError, xsd_error:
return _handle_xsd_errors(request, assets, context, xsd_error,
xsd_data, xsd_file.name)
except Exception, e:
context['errors'] = html_escape(e.message)
return _upload_type_response(request, assets, context)
def _handle_xsd_errors(request, assets, context, xsd_error, xsd_content,
filename):
"""Handle XSD errors. Builds dependency resolver if needed.
Args:
request:
context:
xsd_error:
xsd_content:
filename:
Returns:
def audiofy(match):
url = match.group(0)
url = html_escape(url)
html = u'<audio controls><source src="{url}"><a href="{url}">{url}</a></audio>'.format(url=url)
return self.markdown.htmlStash.store(html, safe=True)
def generate_audit_notification(username, operation_type, obj, changed_fields,
what_changed, is_new_doc=False):
"""
Generate an audit notification on the specific change, if applicable.
This is called during an audit of the object, before the actual save
to the database occurs.
:param username: The user creating the notification.
:type username: str
:param operation_type: The type of operation (i.e. save or delete).
:type operation_type: str
:param obj: The object.
:type obj: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param changed_fields: A list of field names that were changed.
:type changed_fields: list of str
:param message: A message summarizing what changed.
:type message: str
:param is_new_doc: Indicates if the input obj is newly created.
:type is_new_doc: bool
"""
obj_type = obj._meta['crits_type']
supported_notification = __supported_notification_types__.get(obj_type)
# Check if the obj is supported for notifications
if supported_notification is None:
return
if operation_type == "save":
message = "%s updated the following attributes: %s" % (username,
what_changed)
elif operation_type == "delete":
header_description = generate_notification_header(obj)
message = "%s deleted the following: %s" % (username,
header_description)
if is_new_doc:
sources = []
if hasattr(obj, 'source'):
sources = [s.name for s in obj.source]
message = None
target_users = get_subscribed_users(obj_type, obj.id, sources)
header = generate_notification_header(obj)
link_url = None
if hasattr(obj, 'get_details_url'):
link_url = obj.get_details_url()
if header is not None:
header = "New " + header
create_general_notification(username,
target_users,
header,
link_url,
message)
process_result = process_changed_fields(message, changed_fields, obj)
message = process_result.get('message')
source_filter = process_result.get('source_filter')
if message is not None:
message = html_escape(message)
create_notification(obj, username, message, source_filter, NotificationType.ALERT)
def _cast_ballot(self, election_id, username, password, need_login=True, check_user_logged_in=False):
"""
check_user_logged_in looks for the "you're already logged" message
"""
# vote by preparing a ballot via the server-side encryption
response = self.app.post("/helios/elections/%s/encrypt-ballot" % election_id, {
'answers_json': utils.to_json([[1]])})
self.assertContains(response, "answers")
# parse it as an encrypted vote, and re-serialize it
ballot = datatypes.LDObject.fromDict(utils.from_json(response.testbody), type_hint='legacy/EncryptedVote')
encrypted_vote = ballot.serialize()
# cast the ballot
response = self.app.post("/helios/elections/%s/cast" % election_id, {
'encrypted_vote': encrypted_vote})
self.assertRedirects(response, "%s/helios/elections/%s/cast_confirm" % (settings.SECURE_URL_HOST, election_id))
cast_confirm_page = response.follow()
if need_login:
if check_user_logged_in:
self.assertContains(cast_confirm_page, "You are logged in as")
self.assertContains(cast_confirm_page, "requires election-specific credentials")
# set the form
login_form = cast_confirm_page.form
login_form['voter_id'] = username
login_form['password'] = password
cast_confirm_page = login_form.submit()
self.assertRedirects(cast_confirm_page, "/helios/elections/%s/cast_confirm" % election_id)
cast_confirm_page = cast_confirm_page.follow()
# here we should be at the cast-confirm page and logged in
self.assertContains(cast_confirm_page, "I am ")
# confirm the vote, now with the actual form
cast_form = cast_confirm_page.form
if 'status_update' in cast_form.fields.keys():
cast_form['status_update'] = False
response = cast_form.submit()
self.assertRedirects(response, "%s/helios/elections/%s/cast_done" % (settings.URL_HOST, election_id))
# at this point an email should have gone out to the user
# at position num_messages after, since that was the len() before we cast this ballot
email_message = mail.outbox[len(mail.outbox) - 1]
url = re.search('http://[^/]+(/[^ \n]*)', email_message.body).group(1)
# check that we can get at that URL
if not need_login:
# confusing piece: if need_login is True, that means it was a public election
# that required login before casting a ballot.
# so if need_login is False, it was a private election, and we do need to re-login here
# we need to re-login if it's a private election, because all data, including ballots
# is otherwise private
login_page = self.app.get("/helios/elections/%s/password_voter_login" % election_id)
# if we redirected, that's because we can see the page, I think
if login_page.status_int != 302:
login_form = login_page.form
# try with extra spaces
login_form['voter_id'] = ' ' + username + ' '
login_form['password'] = '******' + password + ' '
login_form.submit()
response = self.app.get(url)
self.assertContains(response, ballot.hash)
self.assertContains(response, html_escape(encrypted_vote))
# if we request the redirect to cast_done, the voter should be logged out, but not the user
response = self.app.get("/helios/elections/%s/cast_done" % election_id)
def filter_line(line):
return DIFF_RE.sub(diff_replace, html_escape(line)).replace('\x01', '</span>')
