def format_tz(env, dt, tz=False, format=False):
record_user_timestamp = env.user.sudo().with_context(
tz=tz or env.user.sudo().tz or 'UTC')
timestamp = fields.Datetime.from_string(dt)
ts = fields.Datetime.context_timestamp(record_user_timestamp, timestamp)
# Babel allows to format datetime in a specific language without change locale
# So month 1 = January in English, and janvier in French
# Be aware that the default value for format is 'medium', instead of 'short'
# medium: Jan 5, 2016, 10:20:31 PM | 5 janv. 2016 22:20:31
# short: 1/5/16, 10:20 PM | 5/01/16 22:20
if env.context.get('use_babel'):
# Formatting available here : http://babel.pocoo.org/en/latest/dates.html#date-fields
from babel.dates import format_datetime
return format_datetime(ts,
format or 'medium',
locale=env.context.get("lang") or 'en_US')
if format:
return pycompat.text_type(ts.strftime(format))
else:
lang = env.context.get("lang")
langs = env['res.lang']
if lang:
langs = env['res.lang'].search([("code", "=", lang)])
format_date = langs.date_format or '%B-%d-%Y'
format_time = langs.time_format or '%I-%M %p'
fdate = pycompat.text_type(ts.strftime(format_date))
ftime = pycompat.text_type(ts.strftime(format_time))
return u"%s %s%s" % (fdate, ftime, (u' (%s)' % tz) if tz else u'')
def _read_xls_book(self, book):
sheet = book.sheet_by_index(0)
# emulate Sheet.get_rows for pre-0.9.4
for row in pycompat.imap(sheet.row, range(sheet.nrows)):
values = []
for cell in row:
if cell.ctype is xlrd.XL_CELL_NUMBER:
is_float = cell.value % 1 != 0.0
values.append(
pycompat.text_type(cell.value)
if is_float else pycompat.text_type(int(cell.value)))
elif cell.ctype is xlrd.XL_CELL_DATE:
is_datetime = cell.value % 1 != 0.0
# emulate xldate_as_datetime for pre-0.9.3
dt = datetime.datetime(*xlrd.xldate.xldate_as_tuple(
cell.value, book.datemode))
values.append(
dt.strftime(DEFAULT_SERVER_DATETIME_FORMAT
) if is_datetime else dt.
strftime(DEFAULT_SERVER_DATE_FORMAT))
elif cell.ctype is xlrd.XL_CELL_BOOLEAN:
values.append(u'True' if cell.value else u'False')
elif cell.ctype is xlrd.XL_CELL_ERROR:
raise ValueError(
_("Error cell found while reading XLS/XLSX file: %s") %
xlrd.error_text_from_code.get(
cell.value, "unknown error code %s" % cell.value))
else:
values.append(cell.value)
if any(x for x in values if x.strip()):
yield values
def starttag(self, node, tagname, **attributes):
tagname = pycompat.text_type(tagname).lower()
# extract generic attributes
attrs = {name.lower(): value for name, value in attributes.items()}
attrs.update((name, value) for name, value in node.attributes.items()
if name.startswith('data-'))
prefix = []
postfix = []
# handle possibly multiple ids
assert 'id' not in attrs, "starttag can't be passed a single id attribute, use a list of ids"
ids = node.get('ids', []) + attrs.pop('ids', [])
if ids:
_ids = iter(ids)
attrs['id'] = next(_ids)
postfix.extend(u'<i id="{}"></i>'.format(_id) for _id in _ids)
# set CSS class
classes = set(node.get('classes', []) + attrs.pop('class', '').split())
if classes:
attrs['class'] = u' '.join(classes)
return u'{prefix}<{tag} {attrs}>{postfix}'.format(
prefix=u''.join(prefix),
tag=tagname,
attrs=u' '.join(u'{}="{}"'.format(name, self.attval(value))
for name, value in attrs.items()),
postfix=u''.join(postfix),
)
def test_path(self):
""" Can recursively export fields of m2o via path
"""
record = self.env['export.integer'].create({'value': 42})
self.assertEqual(
self.export(record.id, fields=['value/.id', 'value/value']),
[[pycompat.text_type(record.id), u'42']])
def encode(self, text):
return pycompat.text_type(text).translate({
ord('&'): u'&',
ord('<'): u'<',
ord('"'): u'"',
ord('>'): u'>',
0xa0: u' '
})
def _str_to_selection(self, model, field, value):
# get untranslated values
env = self.with_context(lang=None).env
selection = field.get_description(env)['selection']
for item, label in selection:
label = ustr(label)
labels = [label] + self._get_translations(
('selection', 'model', 'code'), label)
if value == pycompat.text_type(item) or value in labels:
return item, []
raise self._format_import_error(
ValueError,
_(u"Value '%s' not found in selection field '%%(field)s'"), value,
{
'moreinfo': [
_label or pycompat.text_type(item)
for item, _label in selection if _label or item
]
})
def create_token(wizard, partner_id, email):
if context.get("survey_resent_user_input"):
survey_user_input = SurveyUserInput.browse(
context.get("survey_resent_user_input"))
if survey_user_input.state in ('new', 'skip'):
return survey_user_input.token
if context.get("survey_resent_token"):
survey_user_input = SurveyUserInput.search(
[('survey_id', '=', wizard.survey_id.id),
('state', 'in', ['new', 'skip']), '|',
('partner_id', '=', partner_id), ('email', '=', email)],
limit=1)
if survey_user_input:
return survey_user_input.token
if wizard.public != 'email_private':
return None
else:
token = pycompat.text_type(uuid.uuid4())
# create response with token
survey_user_input = SurveyUserInput.create({
'survey_id':
wizard.survey_id.id,
'deadline':
wizard.date_deadline,
'date_create':
fields.Datetime.now(),
'type':
'link',
'state':
'new',
'token':
token,
'partner_id':
partner_id,
'email':
email
})
return survey_user_input.token
def attach(self,
upload=None,
url=None,
disable_optimization=None,
filters=None,
**kwargs):
# the upload argument doesn't allow us to access the files if more than
# one file is uploaded, as upload references the first file
# therefore we have to recover the files from the request object
Attachments = request.env[
'ir.attachment'] # registry for the attachment table
res_model = kwargs.get('res_model', 'ir.ui.view')
if res_model != 'ir.ui.view' and kwargs.get('res_id'):
res_id = int(kwargs['res_id'])
else:
res_id = None
uploads = []
message = None
if not upload: # no image provided, storing the link and the image name
name = url.split("/").pop() # recover filename
datas_fname = name
if filters:
datas_fname = filters + '_' + datas_fname
attachment = Attachments.create({
'name': name,
'datas_fname': datas_fname,
'type': 'url',
'url': url,
'public': res_model == 'ir.ui.view',
'res_id': res_id,
'res_model': res_model,
})
attachment.generate_access_token()
uploads += attachment.read([
'name', 'mimetype', 'checksum', 'url', 'res_id', 'res_model',
'access_token'
])
else: # images provided
try:
attachments = request.env['ir.attachment']
for c_file in request.httprequest.files.getlist('upload'):
data = c_file.read()
try:
image = Image.open(io.BytesIO(data))
w, h = image.size
if w * h > 42e6: # Nokia Lumia 1020 photo resolution
raise ValueError(
u"Image size excessive, uploaded images must be smaller "
u"than 42 million pixel")
if not disable_optimization and image.format in (
'PNG', 'JPEG'):
data = tools.image_save_for_web(image)
except IOError as e:
pass
name = c_file.filename
datas_fname = name
if filters:
datas_fname = filters + '_' + datas_fname
attachment = Attachments.create({
'name':
name,
'datas':
base64.b64encode(data),
'datas_fname':
datas_fname,
'public':
res_model == 'ir.ui.view',
'res_id':
res_id,
'res_model':
res_model,
})
attachment.generate_access_token()
attachments += attachment
uploads += attachments.read([
'name', 'mimetype', 'checksum', 'url', 'res_id',
'res_model', 'access_token'
])
except Exception as e:
logger.exception("Failed to upload image to attachment")
message = pycompat.text_type(e)
return """<script type='text/javascript'>
window.attachments = %s;
window.error = %s;
</script>""" % (json.dumps(uploads), json.dumps(message))
def do(self, fields, columns, options, dryrun=False):
""" Actual execution of the import
:param fields: import mapping: maps each column to a field,
``False`` for the columns to ignore
:type fields: list(str|bool)
:param columns: columns label
:type columns: list(str|bool)
:param dict options:
:param bool dryrun: performs all import operations (and
validations) but rollbacks writes, allows
getting as much errors as possible without
the risk of clobbering the database.
:returns: A list of errors. If the list is empty the import
executed fully and correctly. If the list is
non-empty it contains dicts with 3 keys ``type`` the
type of error (``error|warning``); ``message`` the
error message associated with the error (a string)
and ``record`` the data which failed to import (or
``false`` if that data isn't available or provided)
:rtype: dict(ids: list(int), messages: list({type, message, record}))
"""
self.ensure_one()
self._cr.execute('SAVEPOINT import')
try:
data, import_fields = self._convert_import_data(fields, options)
# Parse date and float field
data = self._parse_import_data(data, import_fields, options)
except ValueError as error:
return {
'messages': [{
'type': 'error',
'message': pycompat.text_type(error),
'record': False,
}]
}
_logger.info('importing %d rows...', len(data))
name_create_enabled_fields = options.pop('name_create_enabled_fields',
{})
model = self.env[self.res_model].with_context(
import_file=True,
name_create_enabled_fields=name_create_enabled_fields)
import_result = model.load(import_fields, data)
_logger.info('done')
# If transaction aborted, RELEASE SAVEPOINT is going to raise
# an InternalError (ROLLBACK should work, maybe). Ignore that.
# TODO: to handle multiple errors, create savepoint around
# write and release it in case of write error (after
# adding error to errors array) => can keep on trying to
# import stuff, and rollback at the end if there is any
# error in the results.
try:
if dryrun:
self._cr.execute('ROLLBACK TO SAVEPOINT import')
# cancel all changes done to the registry/ormcache
self.pool.reset_changes()
else:
self._cr.execute('RELEASE SAVEPOINT import')
except psycopg2.InternalError:
pass
# Insert/Update mapping columns when import complete successfully
if import_result['ids'] and options.get('headers'):
BaseImportMapping = self.env['base_import.mapping']
for index, column_name in enumerate(columns):
if column_name:
# Update to latest selected field
exist_records = BaseImportMapping.search([
('res_model', '=', self.res_model),
('column_name', '=', column_name)
])
if exist_records:
exist_records.write({'field_name': fields[index]})
else:
BaseImportMapping.create({
'res_model': self.res_model,
'column_name': column_name,
'field_name': fields[index]
})
return import_result
def html_sanitize(src,
silent=True,
sanitize_tags=True,
sanitize_attributes=False,
sanitize_style=False,
strip_style=False,
strip_classes=False):
if not src:
return src
src = ustr(src, errors='replace')
# html: remove encoding attribute inside tags
doctype = re.compile(
r'(<[^>]*\s)(encoding=(["\'][^"\']*?["\']|[^\s\n\r>]+)(\s[^>]*|/)?>)',
re.IGNORECASE | re.DOTALL)
src = doctype.sub(u"", src)
logger = logging.getLogger(__name__ + '.html_sanitize')
# html encode email tags
part = re.compile(r"(<(([^a<>]|a[^<>\s])[^<>]*)@[^<>]+>)",
re.IGNORECASE | re.DOTALL)
# remove results containing cite="mid:email_like@address" (ex: blockquote cite)
# cite_except = re.compile(r"^((?!cite[\s]*=['\"]).)*$", re.IGNORECASE)
src = part.sub(
lambda m:
(u'cite=' not in m.group(1) and u'alt=' not in m.group(1) and u'src='
not in m.group(1)) and misc.html_escape(m.group(1)) or m.group(1),
src)
# html encode mako tags <% ... %> to decode them later and keep them alive, otherwise they are stripped by the cleaner
src = src.replace(u'<%', misc.html_escape(u'<%'))
src = src.replace(u'%>', misc.html_escape(u'%>'))
kwargs = {
'page_structure': True,
'style': strip_style, # True = remove style tags/attrs
'sanitize_style': sanitize_style, # True = sanitize styling
'forms': True, # True = remove form tags
'remove_unknown_tags': False,
'comments': False,
'processing_instructions': False
}
if sanitize_tags:
kwargs['allow_tags'] = allowed_tags
if etree.LXML_VERSION >= (2, 3, 1):
# kill_tags attribute has been added in version 2.3.1
kwargs.update({
'kill_tags': tags_to_kill,
'remove_tags': tags_to_remove,
})
else:
kwargs['remove_tags'] = tags_to_kill + tags_to_remove
if sanitize_attributes and etree.LXML_VERSION >= (
3, 1, 0
): # lxml < 3.1.0 does not allow to specify safe_attrs. We keep all attributes in order to keep "style"
if strip_classes:
current_safe_attrs = safe_attrs - frozenset(['class'])
else:
current_safe_attrs = safe_attrs
kwargs.update({
'safe_attrs_only': True,
'safe_attrs': current_safe_attrs,
})
else:
kwargs.update({
'safe_attrs_only': False, # keep oe-data attributes + style
'strip_classes':
strip_classes, # remove classes, even when keeping other attributes
})
try:
# some corner cases make the parser crash (such as <SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT> in test_mail)
cleaner = _Cleaner(**kwargs)
cleaned = cleaner.clean_html(src)
assert isinstance(cleaned, pycompat.text_type)
# MAKO compatibility: $, { and } inside quotes are escaped, preventing correct mako execution
cleaned = cleaned.replace(u'%24', u'$')
cleaned = cleaned.replace(u'%7B', u'{')
cleaned = cleaned.replace(u'%7D', u'}')
cleaned = cleaned.replace(u'%20', u' ')
cleaned = cleaned.replace(u'%5B', u'[')
cleaned = cleaned.replace(u'%5D', u']')
cleaned = cleaned.replace(u'%7C', u'|')
cleaned = cleaned.replace(u'<%', u'<%')
cleaned = cleaned.replace(u'%>', u'%>')
# html considerations so real html content match database value
cleaned.replace(u'\xa0', u' ')
except etree.ParserError as e:
if u'empty' in pycompat.text_type(e):
return u""
if not silent:
raise
logger.warning(u'ParserError obtained when sanitizing %r',
src,
exc_info=True)
cleaned = u'<p>ParserError when sanitizing</p>'
except Exception:
if not silent:
raise
logger.warning(u'unknown error obtained when sanitizing %r',
src,
exc_info=True)
cleaned = u'<p>Unknown error when sanitizing</p>'
# this is ugly, but lxml/etree tostring want to put everything in a 'div' that breaks the editor -> remove that
if cleaned.startswith(u'<div>') and cleaned.endswith(u'</div>'):
cleaned = cleaned[5:-6]
return cleaned
def binary_content(cls,
xmlid=None,
model='ir.attachment',
id=None,
field='datas',
unique=False,
filename=None,
filename_field='datas_fname',
download=False,
mimetype=None,
default_mimetype='application/octet-stream',
access_token=None,
related_id=None,
access_mode=None,
env=None):
""" Get file, attachment or downloadable content
If the ``xmlid`` and ``id`` parameter is omitted, fetches the default value for the
binary field (via ``default_get``), otherwise fetches the field for
that precise record.
:param str xmlid: xmlid of the record
:param str model: name of the model to fetch the binary from
:param int id: id of the record from which to fetch the binary
:param str field: binary field
:param bool unique: add a max-age for the cache control
:param str filename: choose a filename
:param str filename_field: if not create an filename with model-id-field
:param bool download: apply headers to download the file
:param str mimetype: mintype of the field (for headers)
:param related_id: the id of another record used for custom_check
:param access_mode: if truthy, will call custom_check to fetch the object that contains the binary.
:param str default_mimetype: default mintype if no mintype found
:param str access_token: optional token for unauthenticated access
only available for ir.attachment
:param Environment env: by default use request.env
:returns: (status, headers, content)
"""
env = env or request.env
# get object and content
obj = None
if xmlid:
obj = cls._xmlid_to_obj(env, xmlid)
elif id and model in env.registry:
obj = env[model].browse(int(id))
# obj exists
if not obj or not obj.exists() or field not in obj:
return (404, [], None)
# access token grant access
if model == 'ir.attachment' and access_token:
obj = obj.sudo()
if access_mode:
if not cls._check_access_mode(env,
id,
access_mode,
model,
access_token=access_token,
related_id=related_id):
return (403, [], None)
elif not consteq(obj.access_token or u'', access_token):
return (403, [], None)
# check read access
try:
last_update = obj['__last_update']
except AccessError:
return (403, [], None)
status, headers, content = None, [], None
# attachment by url check
module_resource_path = None
if model == 'ir.attachment' and obj.type == 'url' and obj.url:
url_match = re.match("^/(\w+)/(.+)$", obj.url)
if url_match:
module = url_match.group(1)
module_path = get_module_path(module)
module_resource_path = get_resource_path(
module, url_match.group(2))
if module_path and module_resource_path:
module_path = os.path.join(
os.path.normpath(module_path),
'') # join ensures the path ends with '/'
module_resource_path = os.path.normpath(
module_resource_path)
if module_resource_path.startswith(module_path):
with open(module_resource_path, 'rb') as f:
content = base64.b64encode(f.read())
last_update = pycompat.text_type(
os.path.getmtime(module_resource_path))
if not module_resource_path:
module_resource_path = obj.url
if not content:
status = 301
content = module_resource_path
else:
content = obj[field] or ''
# filename
if not filename:
if filename_field in obj:
filename = obj[filename_field]
if not filename and module_resource_path:
filename = os.path.basename(module_resource_path)
if not filename:
filename = "%s-%s-%s" % (obj._name, obj.id, field)
# mimetype
mimetype = 'mimetype' in obj and obj.mimetype or False
if not mimetype:
if filename:
mimetype = mimetypes.guess_type(filename)[0]
if not mimetype and getattr(env[model]._fields[field],
'attachment', False):
# for binary fields, fetch the ir_attachement for mimetype check
attach_mimetype = env['ir.attachment'].search_read(
domain=[('res_model', '=', model), ('res_id', '=', id),
('res_field', '=', field)],
fields=['mimetype'],
limit=1)
mimetype = attach_mimetype and attach_mimetype[0]['mimetype']
if not mimetype:
mimetype = guess_mimetype(base64.b64decode(content),
default=default_mimetype)
# extension
_, existing_extension = os.path.splitext(filename)
if not existing_extension:
extension = mimetypes.guess_extension(mimetype)
if extension:
filename = "%s%s" % (filename, extension)
headers += [('Content-Type', mimetype),
('X-Content-Type-Options', 'nosniff')]
# cache
etag = bool(request) and request.httprequest.headers.get(
'If-None-Match')
retag = '"%s"' % hashlib.md5(
pycompat.to_text(content).encode('utf-8')).hexdigest()
status = status or (304 if etag == retag else 200)
headers.append(('ETag', retag))
headers.append(
('Cache-Control', 'max-age=%s' % (STATIC_CACHE if unique else 0)))
# content-disposition default name
if download:
headers.append(
('Content-Disposition', cls.content_disposition(filename)))
return (status, headers, content)
def attval(self, value, whitespace=re.compile(u'[ \t\n\f\r]+')):
return self.encode(whitespace.sub(u' ', pycompat.text_type(value)))
"""
Store database-specific configuration parameters
"""
import uuid
import logging
from eagle import api, fields, models
from eagle.tools import config, ormcache, mute_logger, pycompat
_logger = logging.getLogger(__name__)
"""
A dictionary holding some configuration parameters to be initialized when the database is created.
"""
_default_parameters = {
"database.secret": lambda: pycompat.text_type(uuid.uuid4()),
"database.uuid": lambda: pycompat.text_type(uuid.uuid1()),
"database.create_date": fields.Datetime.now,
"web.base.url": lambda: "http://localhost:%s" % config.get('http_port'),
"base.login_cooldown_after": lambda: 10,
"base.login_cooldown_duration": lambda: 60,
}
class IrConfigParameter(models.Model):
"""Per-database storage of configuration key-value pairs."""
_name = 'ir.config_parameter'
_description = 'System Parameter'
_rec_name = 'key'
_order = 'key'
def test_huge(self):
self.assertEqual(self.export(2**31 - 1),
[[pycompat.text_type(2**31 - 1)]])