def format_tz(env, dt, tz=False, format=False): record_user_timestamp = env.user.sudo().with_context( tz=tz or env.user.sudo().tz or 'UTC') timestamp = fields.Datetime.from_string(dt) ts = fields.Datetime.context_timestamp(record_user_timestamp, timestamp) # Babel allows to format datetime in a specific language without change locale # So month 1 = January in English, and janvier in French # Be aware that the default value for format is 'medium', instead of 'short' # medium: Jan 5, 2016, 10:20:31 PM | 5 janv. 2016 22:20:31 # short: 1/5/16, 10:20 PM | 5/01/16 22:20 if env.context.get('use_babel'): # Formatting available here : http://babel.pocoo.org/en/latest/dates.html#date-fields from babel.dates import format_datetime return format_datetime(ts, format or 'medium', locale=env.context.get("lang") or 'en_US') if format: return pycompat.text_type(ts.strftime(format)) else: lang = env.context.get("lang") langs = env['res.lang'] if lang: langs = env['res.lang'].search([("code", "=", lang)]) format_date = langs.date_format or '%B-%d-%Y' format_time = langs.time_format or '%I-%M %p' fdate = pycompat.text_type(ts.strftime(format_date)) ftime = pycompat.text_type(ts.strftime(format_time)) return u"%s %s%s" % (fdate, ftime, (u' (%s)' % tz) if tz else u'')
def _read_xls_book(self, book): sheet = book.sheet_by_index(0) # emulate Sheet.get_rows for pre-0.9.4 for row in pycompat.imap(sheet.row, range(sheet.nrows)): values = [] for cell in row: if cell.ctype is xlrd.XL_CELL_NUMBER: is_float = cell.value % 1 != 0.0 values.append( pycompat.text_type(cell.value) if is_float else pycompat.text_type(int(cell.value))) elif cell.ctype is xlrd.XL_CELL_DATE: is_datetime = cell.value % 1 != 0.0 # emulate xldate_as_datetime for pre-0.9.3 dt = datetime.datetime(*xlrd.xldate.xldate_as_tuple( cell.value, book.datemode)) values.append( dt.strftime(DEFAULT_SERVER_DATETIME_FORMAT ) if is_datetime else dt. strftime(DEFAULT_SERVER_DATE_FORMAT)) elif cell.ctype is xlrd.XL_CELL_BOOLEAN: values.append(u'True' if cell.value else u'False') elif cell.ctype is xlrd.XL_CELL_ERROR: raise ValueError( _("Error cell found while reading XLS/XLSX file: %s") % xlrd.error_text_from_code.get( cell.value, "unknown error code %s" % cell.value)) else: values.append(cell.value) if any(x for x in values if x.strip()): yield values
def starttag(self, node, tagname, **attributes): tagname = pycompat.text_type(tagname).lower() # extract generic attributes attrs = {name.lower(): value for name, value in attributes.items()} attrs.update((name, value) for name, value in node.attributes.items() if name.startswith('data-')) prefix = [] postfix = [] # handle possibly multiple ids assert 'id' not in attrs, "starttag can't be passed a single id attribute, use a list of ids" ids = node.get('ids', []) + attrs.pop('ids', []) if ids: _ids = iter(ids) attrs['id'] = next(_ids) postfix.extend(u'<i id="{}"></i>'.format(_id) for _id in _ids) # set CSS class classes = set(node.get('classes', []) + attrs.pop('class', '').split()) if classes: attrs['class'] = u' '.join(classes) return u'{prefix}<{tag} {attrs}>{postfix}'.format( prefix=u''.join(prefix), tag=tagname, attrs=u' '.join(u'{}="{}"'.format(name, self.attval(value)) for name, value in attrs.items()), postfix=u''.join(postfix), )
def test_path(self): """ Can recursively export fields of m2o via path """ record = self.env['export.integer'].create({'value': 42}) self.assertEqual( self.export(record.id, fields=['value/.id', 'value/value']), [[pycompat.text_type(record.id), u'42']])
def encode(self, text): return pycompat.text_type(text).translate({ ord('&'): u'&', ord('<'): u'<', ord('"'): u'"', ord('>'): u'>', 0xa0: u' ' })
def _str_to_selection(self, model, field, value): # get untranslated values env = self.with_context(lang=None).env selection = field.get_description(env)['selection'] for item, label in selection: label = ustr(label) labels = [label] + self._get_translations( ('selection', 'model', 'code'), label) if value == pycompat.text_type(item) or value in labels: return item, [] raise self._format_import_error( ValueError, _(u"Value '%s' not found in selection field '%%(field)s'"), value, { 'moreinfo': [ _label or pycompat.text_type(item) for item, _label in selection if _label or item ] })
def create_token(wizard, partner_id, email): if context.get("survey_resent_user_input"): survey_user_input = SurveyUserInput.browse( context.get("survey_resent_user_input")) if survey_user_input.state in ('new', 'skip'): return survey_user_input.token if context.get("survey_resent_token"): survey_user_input = SurveyUserInput.search( [('survey_id', '=', wizard.survey_id.id), ('state', 'in', ['new', 'skip']), '|', ('partner_id', '=', partner_id), ('email', '=', email)], limit=1) if survey_user_input: return survey_user_input.token if wizard.public != 'email_private': return None else: token = pycompat.text_type(uuid.uuid4()) # create response with token survey_user_input = SurveyUserInput.create({ 'survey_id': wizard.survey_id.id, 'deadline': wizard.date_deadline, 'date_create': fields.Datetime.now(), 'type': 'link', 'state': 'new', 'token': token, 'partner_id': partner_id, 'email': email }) return survey_user_input.token
def attach(self, upload=None, url=None, disable_optimization=None, filters=None, **kwargs): # the upload argument doesn't allow us to access the files if more than # one file is uploaded, as upload references the first file # therefore we have to recover the files from the request object Attachments = request.env[ 'ir.attachment'] # registry for the attachment table res_model = kwargs.get('res_model', 'ir.ui.view') if res_model != 'ir.ui.view' and kwargs.get('res_id'): res_id = int(kwargs['res_id']) else: res_id = None uploads = [] message = None if not upload: # no image provided, storing the link and the image name name = url.split("/").pop() # recover filename datas_fname = name if filters: datas_fname = filters + '_' + datas_fname attachment = Attachments.create({ 'name': name, 'datas_fname': datas_fname, 'type': 'url', 'url': url, 'public': res_model == 'ir.ui.view', 'res_id': res_id, 'res_model': res_model, }) attachment.generate_access_token() uploads += attachment.read([ 'name', 'mimetype', 'checksum', 'url', 'res_id', 'res_model', 'access_token' ]) else: # images provided try: attachments = request.env['ir.attachment'] for c_file in request.httprequest.files.getlist('upload'): data = c_file.read() try: image = Image.open(io.BytesIO(data)) w, h = image.size if w * h > 42e6: # Nokia Lumia 1020 photo resolution raise ValueError( u"Image size excessive, uploaded images must be smaller " u"than 42 million pixel") if not disable_optimization and image.format in ( 'PNG', 'JPEG'): data = tools.image_save_for_web(image) except IOError as e: pass name = c_file.filename datas_fname = name if filters: datas_fname = filters + '_' + datas_fname attachment = Attachments.create({ 'name': name, 'datas': base64.b64encode(data), 'datas_fname': datas_fname, 'public': res_model == 'ir.ui.view', 'res_id': res_id, 'res_model': res_model, }) attachment.generate_access_token() attachments += attachment uploads += attachments.read([ 'name', 'mimetype', 'checksum', 'url', 'res_id', 'res_model', 'access_token' ]) except Exception as e: logger.exception("Failed to upload image to attachment") message = pycompat.text_type(e) return """<script type='text/javascript'> window.attachments = %s; window.error = %s; </script>""" % (json.dumps(uploads), json.dumps(message))
def do(self, fields, columns, options, dryrun=False): """ Actual execution of the import :param fields: import mapping: maps each column to a field, ``False`` for the columns to ignore :type fields: list(str|bool) :param columns: columns label :type columns: list(str|bool) :param dict options: :param bool dryrun: performs all import operations (and validations) but rollbacks writes, allows getting as much errors as possible without the risk of clobbering the database. :returns: A list of errors. If the list is empty the import executed fully and correctly. If the list is non-empty it contains dicts with 3 keys ``type`` the type of error (``error|warning``); ``message`` the error message associated with the error (a string) and ``record`` the data which failed to import (or ``false`` if that data isn't available or provided) :rtype: dict(ids: list(int), messages: list({type, message, record})) """ self.ensure_one() self._cr.execute('SAVEPOINT import') try: data, import_fields = self._convert_import_data(fields, options) # Parse date and float field data = self._parse_import_data(data, import_fields, options) except ValueError as error: return { 'messages': [{ 'type': 'error', 'message': pycompat.text_type(error), 'record': False, }] } _logger.info('importing %d rows...', len(data)) name_create_enabled_fields = options.pop('name_create_enabled_fields', {}) model = self.env[self.res_model].with_context( import_file=True, name_create_enabled_fields=name_create_enabled_fields) import_result = model.load(import_fields, data) _logger.info('done') # If transaction aborted, RELEASE SAVEPOINT is going to raise # an InternalError (ROLLBACK should work, maybe). Ignore that. # TODO: to handle multiple errors, create savepoint around # write and release it in case of write error (after # adding error to errors array) => can keep on trying to # import stuff, and rollback at the end if there is any # error in the results. try: if dryrun: self._cr.execute('ROLLBACK TO SAVEPOINT import') # cancel all changes done to the registry/ormcache self.pool.reset_changes() else: self._cr.execute('RELEASE SAVEPOINT import') except psycopg2.InternalError: pass # Insert/Update mapping columns when import complete successfully if import_result['ids'] and options.get('headers'): BaseImportMapping = self.env['base_import.mapping'] for index, column_name in enumerate(columns): if column_name: # Update to latest selected field exist_records = BaseImportMapping.search([ ('res_model', '=', self.res_model), ('column_name', '=', column_name) ]) if exist_records: exist_records.write({'field_name': fields[index]}) else: BaseImportMapping.create({ 'res_model': self.res_model, 'column_name': column_name, 'field_name': fields[index] }) return import_result
def html_sanitize(src, silent=True, sanitize_tags=True, sanitize_attributes=False, sanitize_style=False, strip_style=False, strip_classes=False): if not src: return src src = ustr(src, errors='replace') # html: remove encoding attribute inside tags doctype = re.compile( r'(<[^>]*\s)(encoding=(["\'][^"\']*?["\']|[^\s\n\r>]+)(\s[^>]*|/)?>)', re.IGNORECASE | re.DOTALL) src = doctype.sub(u"", src) logger = logging.getLogger(__name__ + '.html_sanitize') # html encode email tags part = re.compile(r"(<(([^a<>]|a[^<>\s])[^<>]*)@[^<>]+>)", re.IGNORECASE | re.DOTALL) # remove results containing cite="mid:email_like@address" (ex: blockquote cite) # cite_except = re.compile(r"^((?!cite[\s]*=['\"]).)*$", re.IGNORECASE) src = part.sub( lambda m: (u'cite=' not in m.group(1) and u'alt=' not in m.group(1) and u'src=' not in m.group(1)) and misc.html_escape(m.group(1)) or m.group(1), src) # html encode mako tags <% ... %> to decode them later and keep them alive, otherwise they are stripped by the cleaner src = src.replace(u'<%', misc.html_escape(u'<%')) src = src.replace(u'%>', misc.html_escape(u'%>')) kwargs = { 'page_structure': True, 'style': strip_style, # True = remove style tags/attrs 'sanitize_style': sanitize_style, # True = sanitize styling 'forms': True, # True = remove form tags 'remove_unknown_tags': False, 'comments': False, 'processing_instructions': False } if sanitize_tags: kwargs['allow_tags'] = allowed_tags if etree.LXML_VERSION >= (2, 3, 1): # kill_tags attribute has been added in version 2.3.1 kwargs.update({ 'kill_tags': tags_to_kill, 'remove_tags': tags_to_remove, }) else: kwargs['remove_tags'] = tags_to_kill + tags_to_remove if sanitize_attributes and etree.LXML_VERSION >= ( 3, 1, 0 ): # lxml < 3.1.0 does not allow to specify safe_attrs. We keep all attributes in order to keep "style" if strip_classes: current_safe_attrs = safe_attrs - frozenset(['class']) else: current_safe_attrs = safe_attrs kwargs.update({ 'safe_attrs_only': True, 'safe_attrs': current_safe_attrs, }) else: kwargs.update({ 'safe_attrs_only': False, # keep oe-data attributes + style 'strip_classes': strip_classes, # remove classes, even when keeping other attributes }) try: # some corner cases make the parser crash (such as <SCRIPT/XSS SRC=\"http://ha.ckers.org/xss.js\"></SCRIPT> in test_mail) cleaner = _Cleaner(**kwargs) cleaned = cleaner.clean_html(src) assert isinstance(cleaned, pycompat.text_type) # MAKO compatibility: $, { and } inside quotes are escaped, preventing correct mako execution cleaned = cleaned.replace(u'%24', u'$') cleaned = cleaned.replace(u'%7B', u'{') cleaned = cleaned.replace(u'%7D', u'}') cleaned = cleaned.replace(u'%20', u' ') cleaned = cleaned.replace(u'%5B', u'[') cleaned = cleaned.replace(u'%5D', u']') cleaned = cleaned.replace(u'%7C', u'|') cleaned = cleaned.replace(u'<%', u'<%') cleaned = cleaned.replace(u'%>', u'%>') # html considerations so real html content match database value cleaned.replace(u'\xa0', u' ') except etree.ParserError as e: if u'empty' in pycompat.text_type(e): return u"" if not silent: raise logger.warning(u'ParserError obtained when sanitizing %r', src, exc_info=True) cleaned = u'<p>ParserError when sanitizing</p>' except Exception: if not silent: raise logger.warning(u'unknown error obtained when sanitizing %r', src, exc_info=True) cleaned = u'<p>Unknown error when sanitizing</p>' # this is ugly, but lxml/etree tostring want to put everything in a 'div' that breaks the editor -> remove that if cleaned.startswith(u'<div>') and cleaned.endswith(u'</div>'): cleaned = cleaned[5:-6] return cleaned
def binary_content(cls, xmlid=None, model='ir.attachment', id=None, field='datas', unique=False, filename=None, filename_field='datas_fname', download=False, mimetype=None, default_mimetype='application/octet-stream', access_token=None, related_id=None, access_mode=None, env=None): """ Get file, attachment or downloadable content If the ``xmlid`` and ``id`` parameter is omitted, fetches the default value for the binary field (via ``default_get``), otherwise fetches the field for that precise record. :param str xmlid: xmlid of the record :param str model: name of the model to fetch the binary from :param int id: id of the record from which to fetch the binary :param str field: binary field :param bool unique: add a max-age for the cache control :param str filename: choose a filename :param str filename_field: if not create an filename with model-id-field :param bool download: apply headers to download the file :param str mimetype: mintype of the field (for headers) :param related_id: the id of another record used for custom_check :param access_mode: if truthy, will call custom_check to fetch the object that contains the binary. :param str default_mimetype: default mintype if no mintype found :param str access_token: optional token for unauthenticated access only available for ir.attachment :param Environment env: by default use request.env :returns: (status, headers, content) """ env = env or request.env # get object and content obj = None if xmlid: obj = cls._xmlid_to_obj(env, xmlid) elif id and model in env.registry: obj = env[model].browse(int(id)) # obj exists if not obj or not obj.exists() or field not in obj: return (404, [], None) # access token grant access if model == 'ir.attachment' and access_token: obj = obj.sudo() if access_mode: if not cls._check_access_mode(env, id, access_mode, model, access_token=access_token, related_id=related_id): return (403, [], None) elif not consteq(obj.access_token or u'', access_token): return (403, [], None) # check read access try: last_update = obj['__last_update'] except AccessError: return (403, [], None) status, headers, content = None, [], None # attachment by url check module_resource_path = None if model == 'ir.attachment' and obj.type == 'url' and obj.url: url_match = re.match("^/(\w+)/(.+)$", obj.url) if url_match: module = url_match.group(1) module_path = get_module_path(module) module_resource_path = get_resource_path( module, url_match.group(2)) if module_path and module_resource_path: module_path = os.path.join( os.path.normpath(module_path), '') # join ensures the path ends with '/' module_resource_path = os.path.normpath( module_resource_path) if module_resource_path.startswith(module_path): with open(module_resource_path, 'rb') as f: content = base64.b64encode(f.read()) last_update = pycompat.text_type( os.path.getmtime(module_resource_path)) if not module_resource_path: module_resource_path = obj.url if not content: status = 301 content = module_resource_path else: content = obj[field] or '' # filename if not filename: if filename_field in obj: filename = obj[filename_field] if not filename and module_resource_path: filename = os.path.basename(module_resource_path) if not filename: filename = "%s-%s-%s" % (obj._name, obj.id, field) # mimetype mimetype = 'mimetype' in obj and obj.mimetype or False if not mimetype: if filename: mimetype = mimetypes.guess_type(filename)[0] if not mimetype and getattr(env[model]._fields[field], 'attachment', False): # for binary fields, fetch the ir_attachement for mimetype check attach_mimetype = env['ir.attachment'].search_read( domain=[('res_model', '=', model), ('res_id', '=', id), ('res_field', '=', field)], fields=['mimetype'], limit=1) mimetype = attach_mimetype and attach_mimetype[0]['mimetype'] if not mimetype: mimetype = guess_mimetype(base64.b64decode(content), default=default_mimetype) # extension _, existing_extension = os.path.splitext(filename) if not existing_extension: extension = mimetypes.guess_extension(mimetype) if extension: filename = "%s%s" % (filename, extension) headers += [('Content-Type', mimetype), ('X-Content-Type-Options', 'nosniff')] # cache etag = bool(request) and request.httprequest.headers.get( 'If-None-Match') retag = '"%s"' % hashlib.md5( pycompat.to_text(content).encode('utf-8')).hexdigest() status = status or (304 if etag == retag else 200) headers.append(('ETag', retag)) headers.append( ('Cache-Control', 'max-age=%s' % (STATIC_CACHE if unique else 0))) # content-disposition default name if download: headers.append( ('Content-Disposition', cls.content_disposition(filename))) return (status, headers, content)
def attval(self, value, whitespace=re.compile(u'[ \t\n\f\r]+')): return self.encode(whitespace.sub(u' ', pycompat.text_type(value)))
""" Store database-specific configuration parameters """ import uuid import logging from eagle import api, fields, models from eagle.tools import config, ormcache, mute_logger, pycompat _logger = logging.getLogger(__name__) """ A dictionary holding some configuration parameters to be initialized when the database is created. """ _default_parameters = { "database.secret": lambda: pycompat.text_type(uuid.uuid4()), "database.uuid": lambda: pycompat.text_type(uuid.uuid1()), "database.create_date": fields.Datetime.now, "web.base.url": lambda: "http://localhost:%s" % config.get('http_port'), "base.login_cooldown_after": lambda: 10, "base.login_cooldown_duration": lambda: 60, } class IrConfigParameter(models.Model): """Per-database storage of configuration key-value pairs.""" _name = 'ir.config_parameter' _description = 'System Parameter' _rec_name = 'key' _order = 'key'
def test_huge(self): self.assertEqual(self.export(2**31 - 1), [[pycompat.text_type(2**31 - 1)]])