def upload(self, dir, datafile, **kwargs):
error = ''
fn = ''
if datafile:
path = Path(op.join(datapath, dir))
if not path:
path.make()
fn = path + datafile.filename
if not fn.is_legal:
error = "'%s' is not legal"
if fn and 'overwrite' not in kwargs:
error = "'%s' exists already, if you want to overwrite the old version, check allow overwrite" % fn.name
# Buffer file for first check encoding and secondly upload file
with BytesIO(datafile.file.read()) as filebuffer:
# determine file encodings
result = chardet.detect(filebuffer.read())
# Reset file buffer
filebuffer.seek(0)
# if chardet can determine file encoding, check it and warn respectively
# otherwise state not detecting
# TODO: chardet cannot determine sufficent amount of encodings, such as utf-16-le
if result['encoding']:
file_encoding = result['encoding'].lower()
# TODO: outsource valid encodings
if not (file_encoding in ['utf-8', 'ascii']
or 'utf-8' in file_encoding):
log.error("WARNING: encoding of file {} is {}".format(
datafile.filename, file_encoding))
else:
msg = "WARNING: encoding of file {} is not detectable".format(
datafile.filename)
log.error(msg)
try:
write_to_file(fn.absolute, filebuffer)
fn.setownergroup()
except:
error += '\n' + traceback()
print(error)
if "uploadimport" in kwargs and not error:
url = '/download/to_db?filename=' + escape(fn.href)
else:
url = '/download?dir=' + escape(dir)
if error:
url += '&error=' + escape(error)
raise web.HTTPRedirect(url)
def email_body(body):
#body = str(body)
#if type(body) == str:
# body = body.decode('utf8')
body = body.replace('\r\n', '\n')
# fix issue where quoted text immediately after quoted text at start of
# quoting
body = body.replace(':\n>', ':\n\n>')
# in quoted sections seem to get line continuations with =
body = body.replace('=\n', '')
# incredibly ugly hack to deal with <*****@*****.**> in message body
# (does not get handled by markdown for some reason ...)
import re
body = re.sub('<isitopen.okfn.org\s*>', '<isitopen.okfn.org>', body)
# body = body.replace('<isitopen.okfn.org>', '<isitopen.okfn.org>')
#return ""
#return markdown(repr(type(body)).replace("<", "<") + body[:10])
# Todo: Fix this, it screws up when unicode is in the enquiry body.
try:
out = genshi.HTML(markdown(body))
except:
out = '<p><strong>We had problems prettifying the email you are trying to display -- probably due to broken HTML in it!</strong></p>\n\n'
try:
out += unicode(genshi.escape(body))
except:
pass
out = genshi.HTML(out)
return out
def newfolder(self, dir, newfolder):
error = ''
if newfolder:
if ' ' in newfolder:
error = "The folder name may not include a space!"
else:
try:
path = Path(op.join(datapath, dir, newfolder))
if not path:
path.make()
path.setownergroup()
else:
error = "Folder %s exists already!" % newfolder
except:
error = traceback()
else:
error = 'Forgotten to give your new folder a name?'
url = '/download?dir=' + escape(dir)
if error:
url += '&error=' + escape(error)
return self.index(dir=dir, error=error)
def logimport(self, filename, kwargs, import_with_class=LogbookImport):
"""
:param filename:
:param kwargs:
:param import_with_class:
:return:
"""
import dataimport.importlog as il
t0 = time.time()
absfile = web.abspath(filename.strip('/'))
path = Path(absfile)
import dataimport as di
error = web.markdown(di.checkimport(path.absolute))
config = None
if import_with_class == ManualMeasurementsImport:
config = ManualMeasurementsImport.from_file(path.absolute)
print("path = %s;\nabsfile = %s" % (path, absfile))
from cherrypy import log
log("Import with class %s" % import_with_class.__name__)
li = import_with_class(absfile, web.user(), config=config)
# TODO: Sometimes this is causing a delay
logs, cancommit = li('commit' in kwargs)
# TODO: REFACTORING FOR MAINTAINABILITY
t1 = time.time()
log("Imported in %.2f s" % (t1 - t0))
if 'commit' in kwargs and cancommit:
di.savetoimports(absfile, web.user(), ["_various_as_its_manual"])
raise web.HTTPRedirect('/download?dir=' + escape(path.up()))
else:
return web.render('logimport.html', filename=path, logs=logs,
cancommit=cancommit, error=error)\
.render('html', doctype='html')
arglist = args.strip().split('|')
# Arg 0 is the slug of the photo
photo = page.photos.get(slug=arglist[0])
attrs = parse_img_attributes(photo, arglist)
try:
thumb = photo.get_thumb(attrs.size[0], attrs.size[1], 'Stretch')
except IOError, e:
# If an image is missing from the media folder, don't kill the page
log.warn('Cannot render photo %s for %s %s' % (arglist[0],
page.get_absolute_url(), e))
thumb = None
align = attrs.get('align', None)
caption = None
if 'caption' in attrs:
caption = escape(attrs.caption)
type = attrs.get('type', None)
alt_text = escape(attrs.alt) if 'alt' in attrs else caption
has_frame = type in ('thumb', 'thumbnail', 'frame')
table_frame = has_frame and caption and not editing
img_frame = has_frame and not table_frame
# Build the image element first.
img_src = thumb.get_absolute_url() if thumb is not None else ''
if editing:
# Save the original args in the url as a dummy query string.
# In google chrome and windows safari, only the src attribute
# of a dragged image survives.
img_src += '?wiki_img_args=' + urllib.quote(args.strip())
img_attrs = [
'src="%s"' % escape(img_src),
def diff_blocks(fromlines,
tolines,
context=None,
tabwidth=8,
ignore_blank_lines=0,
ignore_case=0,
ignore_space_changes=0):
"""Return an array that is adequate for adding to the data dictionary
See `get_filtered_hunks` for the parameter descriptions.
See also the diff_div.html template.
"""
type_map = {
'replace': 'mod',
'delete': 'rem',
'insert': 'add',
'equal': 'unmod'
}
space_re = re.compile(' ( +)|^ ')
def htmlify(match):
div, mod = divmod(len(match.group(0)), 2)
return div * ' ' + mod * ' '
def markup_intraline_changes(opcodes):
for tag, i1, i2, j1, j2 in opcodes:
if tag == 'replace' and i2 - i1 == j2 - j1:
for i in range(i2 - i1):
fromline, toline = fromlines[i1 + i], tolines[j1 + i]
(start, end) = get_change_extent(fromline, toline)
if start != 0 or end != 0:
last = end + len(fromline)
fromlines[i1 + i] = (fromline[:start] + '\0' +
fromline[start:last] + '\1' +
fromline[last:])
last = end + len(toline)
tolines[j1 + i] = (toline[:start] + '\0' +
toline[start:last] + '\1' +
toline[last:])
yield tag, i1, i2, j1, j2
changes = []
for group in get_filtered_hunks(fromlines, tolines, context,
ignore_blank_lines, ignore_case,
ignore_space_changes):
blocks = []
last_tag = None
for tag, i1, i2, j1, j2 in markup_intraline_changes(group):
if tag != last_tag:
blocks.append({
'type': type_map[tag],
'base': {
'offset': i1,
'lines': []
},
'changed': {
'offset': j1,
'lines': []
}
})
if tag == 'equal':
for line in fromlines[i1:i2]:
line = line.expandtabs(tabwidth)
line = space_re.sub(htmlify, escape(line, quotes=False))
blocks[-1]['base']['lines'].append(Markup(unicode(line)))
for line in tolines[j1:j2]:
line = line.expandtabs(tabwidth)
line = space_re.sub(htmlify, escape(line, quotes=False))
blocks[-1]['changed']['lines'].append(Markup(
unicode(line)))
else:
if tag in ('replace', 'delete'):
for line in fromlines[i1:i2]:
line = expandtabs(line, tabwidth, '\0\1')
line = escape(line, quotes=False)
line = '<del>'.join([
space_re.sub(htmlify, seg)
for seg in line.split('\0')
])
line = line.replace('\1', '</del>')
blocks[-1]['base']['lines'].append(
Markup(unicode(line)))
if tag in ('replace', 'insert'):
for line in tolines[j1:j2]:
line = expandtabs(line, tabwidth, '\0\1')
line = escape(line, quotes=False)
line = '<ins>'.join([
space_re.sub(htmlify, seg)
for seg in line.split('\0')
])
line = line.replace('\1', '</ins>')
blocks[-1]['changed']['lines'].append(
Markup(unicode(line)))
changes.append(blocks)
return changes
def diff_blocks(fromlines, tolines, context=None, tabwidth=8,
ignore_blank_lines=0, ignore_case=0, ignore_space_changes=0):
"""Return an array that is adequate for adding to the data dictionary
See `get_filtered_hunks` for the parameter descriptions.
See also the diff_div.html template.
"""
type_map = {'replace': 'mod', 'delete': 'rem', 'insert': 'add',
'equal': 'unmod'}
space_re = re.compile(' ( +)|^ ')
def htmlify(match):
div, mod = divmod(len(match.group(0)), 2)
return div * ' ' + mod * ' '
def markup_intraline_changes(opcodes):
for tag, i1, i2, j1, j2 in opcodes:
if tag == 'replace' and i2 - i1 == j2 - j1:
for i in range(i2 - i1):
fromline, toline = fromlines[i1 + i], tolines[j1 + i]
(start, end) = get_change_extent(fromline, toline)
if start != 0 or end != 0:
last = end + len(fromline)
fromlines[i1 + i] = (
fromline[:start] + '\0' + fromline[start:last] +
'\1' + fromline[last:])
last = end+len(toline)
tolines[j1 + i] = (
toline[:start] + '\0' + toline[start:last] +
'\1' + toline[last:])
yield tag, i1, i2, j1, j2
changes = []
for group in get_filtered_hunks(fromlines, tolines, context,
ignore_blank_lines, ignore_case,
ignore_space_changes):
blocks = []
last_tag = None
for tag, i1, i2, j1, j2 in markup_intraline_changes(group):
if tag != last_tag:
blocks.append({'type': type_map[tag],
'base': {'offset': i1, 'lines': []},
'changed': {'offset': j1, 'lines': []}})
if tag == 'equal':
for line in fromlines[i1:i2]:
line = line.expandtabs(tabwidth)
line = space_re.sub(htmlify, escape(line, quotes=False))
blocks[-1]['base']['lines'].append(Markup(unicode(line)))
for line in tolines[j1:j2]:
line = line.expandtabs(tabwidth)
line = space_re.sub(htmlify, escape(line, quotes=False))
blocks[-1]['changed']['lines'].append(Markup(unicode(line)))
else:
if tag in ('replace', 'delete'):
for line in fromlines[i1:i2]:
line = expandtabs(line, tabwidth, '\0\1')
line = escape(line, quotes=False)
line = '<del>'.join([space_re.sub(htmlify, seg)
for seg in line.split('\0')])
line = line.replace('\1', '</del>')
blocks[-1]['base']['lines'].append(
Markup(unicode(line)))
if tag in ('replace', 'insert'):
for line in tolines[j1:j2]:
line = expandtabs(line, tabwidth, '\0\1')
line = escape(line, quotes=False)
line = '<ins>'.join([space_re.sub(htmlify, seg)
for seg in line.split('\0')])
line = line.replace('\1', '</ins>')
blocks[-1]['changed']['lines'].append(
Markup(unicode(line)))
changes.append(blocks)
return changes
class TransposingElementFactory(ElementFactory):
"""A `genshi.builder.ElementFactory` which applies `func` to the
named attributes before creating a `genshi.builder.Element`.
"""
def __init__(self, func, namespace=None):
ElementFactory.__init__(self, namespace=namespace)
self.func = func
def __getattr__(self, name):
return ElementFactory.__getattr__(self, self.func(name))
html = TransposingElementFactory(str.lower)
try:
escape('', False) # detect genshi:#439 on Genshi 0.6 with speedups
except TypeError:
_escape = escape
def escape(text, quotes=True):
if text:
return _escape(text, quotes=quotes)
else:
return Markup(u'')
def plaintext(text, keeplinebreaks=True):
"""Extract the text elements from (X)HTML content
:param text: `unicode` or `genshi.builder.Fragment`
:param keeplinebreaks: optionally keep linebreaks