def test_mimetype_svg(self):
content = base64.b64decode(SVG)
mimetype = guess_mimetype(content, default='test')
self.assertTrue(mimetype.startswith('image/svg'))
# Tests that whitespace padded SVG are not detected as SVG
mimetype = guess_mimetype(b" " + content, default='test')
self.assertNotIn("svg", mimetype)
def guess_extension(filename=None, mimetype=None, binary=None):
extension = filename and os.path.splitext(filename)[1][1:].strip().lower()
if not extension and mimetype:
extension = mimetypes.guess_extension(mimetype)[1:].strip().lower()
if not extension and binary:
mimetype = guess_mimetype(binary, default="")
extension = mimetypes.guess_extension(mimetype)[1:].strip().lower()
return extension
def _compute_mimetype(self, values):
""" compute the mimetype of the given values
:param values : dict of values to create or write an ir_attachment
:return mime : string indicating the mimetype, or application/octet-stream by default
"""
mimetype = None
if values.get('mimetype'):
mimetype = values['mimetype']
if not mimetype and values.get('datas_fname'):
mimetype = mimetypes.guess_type(values['datas_fname'])[0]
if not mimetype and values.get('url'):
mimetype = mimetypes.guess_type(values['url'])[0]
if values.get('datas') and (not mimetype or mimetype == 'application/octet-stream'):
mimetype = guess_mimetype(base64.b64decode(values['datas']))
return mimetype or 'application/octet-stream'
def _read_file(self, options):
""" Dispatch to specific method to read file content, according to its mimetype or file type
:param options : dict of reading options (quoting, separator, ...)
"""
self.ensure_one()
# guess mimetype from file content
mimetype = guess_mimetype(self.file)
(file_extension, handler,
req) = FILE_TYPE_DICT.get(mimetype, (None, None, None))
if handler:
try:
return getattr(self, '_read_' + file_extension)(options)
except Exception:
_logger.warn(
"Failed to read file '%s' (transient id %d) using guessed mimetype %s",
self.file_name or '<unknown>', self.id, mimetype)
# try reading with user-provided mimetype
(file_extension, handler,
req) = FILE_TYPE_DICT.get(self.file_type, (None, None, None))
if handler:
try:
return getattr(self, '_read_' + file_extension)(options)
except Exception:
_logger.warn(
"Failed to read file '%s' (transient id %d) using user-provided mimetype %s",
self.file_name or '<unknown>', self.id, self.file_type)
# fallback on file extensions as mime types can be unreliable (e.g.
# software setting incorrect mime types, or non-installed software
# leading to browser not sending mime types)
if self.file_name:
p, ext = os.path.splitext(self.file_name)
if ext in EXTENSIONS:
try:
return getattr(self, '_read_' + ext[1:])(options)
except Exception:
_logger.warn(
"Failed to read file '%s' (transient id %s) using file extension",
self.file_name, self.id)
if req:
raise ImportError(
_("Unable to load \"{extension}\" file: requires Python module \"{modname}\""
).format(extension=file_extension, modname=req))
raise ValueError(
_("Unsupported file format \"{}\", import only supports CSV, ODS, XLS and XLSX"
).format(self.file_type))
def _binary_ir_attachment_redirect_content(
cls, record, default_mimetype='application/octet-stream'):
# mainly used for theme images attachemnts
status = content = filename = filehash = None
mimetype = getattr(record, 'mimetype', False)
if record.type == 'url' and record.url:
# if url in in the form /somehint server locally
url_match = re.match("^/(\w+)/(.+)$", record.url)
if url_match:
module = url_match.group(1)
module_path = get_module_path(module)
module_resource_path = get_resource_path(
module, url_match.group(2))
if module_path and module_resource_path:
module_path = os.path.join(
os.path.normpath(module_path),
'') # join ensures the path ends with '/'
module_resource_path = os.path.normpath(
module_resource_path)
if module_resource_path.startswith(module_path):
with open(module_resource_path, 'rb') as f:
content = base64.b64encode(f.read())
status = 200
filename = os.path.basename(module_resource_path)
mimetype = guess_mimetype(base64.b64decode(content),
default=default_mimetype)
filehash = '"%s"' % hashlib.md5(
pycompat.to_text(content).encode(
'utf-8')).hexdigest()
if not content:
status = 301
content = record.url
return status, content, filename, mimetype, filehash
def test_xls(self):
self.assertEqual(
guess_mimetype(contents('xls')),
'application/vnd.ms-excel'
)
def binary_content(cls,
xmlid=None,
model='ir.attachment',
id=None,
field='datas',
unique=False,
filename=None,
filename_field='datas_fname',
download=False,
mimetype=None,
default_mimetype='application/octet-stream',
access_token=None,
env=None):
""" Get file, attachment or downloadable content
If the ``xmlid`` and ``id`` parameter is omitted, fetches the default value for the
binary field (via ``default_get``), otherwise fetches the field for
that precise record.
:param str xmlid: xmlid of the record
:param str model: name of the model to fetch the binary from
:param int id: id of the record from which to fetch the binary
:param str field: binary field
:param bool unique: add a max-age for the cache control
:param str filename: choose a filename
:param str filename_field: if not create an filename with model-id-field
:param bool download: apply headers to download the file
:param str mimetype: mintype of the field (for headers)
:param str default_mimetype: default mintype if no mintype found
:param str access_token: optional token for unauthenticated access
only available for ir.attachment
:param Environment env: by default use request.env
:returns: (status, headers, content)
"""
env = env or request.env
# get object and content
obj = None
if xmlid:
obj = env.ref(xmlid, False)
elif id and model == 'ir.attachment' and access_token:
obj = env[model].sudo().browse(int(id))
if not consteq(obj.access_token, access_token):
return (403, [], None)
elif id and model in env.registry:
obj = env[model].browse(int(id))
# obj exists
if not obj or not obj.exists() or field not in obj:
return (404, [], None)
# check read access
try:
last_update = obj['__last_update']
except AccessError:
return (403, [], None)
status, headers, content = None, [], None
# attachment by url check
module_resource_path = None
if model == 'ir.attachment' and obj.type == 'url' and obj.url:
url_match = re.match("^/(\w+)/(.+)$", obj.url)
if url_match:
module = url_match.group(1)
module_path = get_module_path(module)
module_resource_path = get_resource_path(
module, url_match.group(2))
if module_path and module_resource_path:
module_path = os.path.join(
os.path.normpath(module_path),
'') # join ensures the path ends with '/'
module_resource_path = os.path.normpath(
module_resource_path)
if module_resource_path.startswith(module_path):
with open(module_resource_path, 'rb') as f:
content = base64.b64encode(f.read())
last_update = pycompat.text_type(
os.path.getmtime(module_resource_path))
if not module_resource_path:
module_resource_path = obj.url
if not content:
status = 301
content = module_resource_path
else:
content = obj[field] or ''
# filename
if not filename:
if filename_field in obj:
filename = obj[filename_field]
elif module_resource_path:
filename = os.path.basename(module_resource_path)
else:
filename = "%s-%s-%s" % (obj._name, obj.id, field)
# mimetype
mimetype = 'mimetype' in obj and obj.mimetype or False
if not mimetype:
if filename:
mimetype = mimetypes.guess_type(filename)[0]
if not mimetype and getattr(env[model]._fields[field],
'attachment', False):
# for binary fields, fetch the ir_attachement for mimetype check
attach_mimetype = env['ir.attachment'].search_read(
domain=[('res_model', '=', model), ('res_id', '=', id),
('res_field', '=', field)],
fields=['mimetype'],
limit=1)
mimetype = attach_mimetype and attach_mimetype[0]['mimetype']
if not mimetype:
mimetype = guess_mimetype(base64.b64decode(content),
default=default_mimetype)
headers += [('Content-Type', mimetype),
('X-Content-Type-Options', 'nosniff')]
# cache
etag = bool(request) and request.httprequest.headers.get(
'If-None-Match')
retag = '"%s"' % hashlib.md5(
pycompat.to_text(content).encode('utf-8')).hexdigest()
status = status or (304 if etag == retag else 200)
headers.append(('ETag', retag))
headers.append(
('Cache-Control', 'max-age=%s' % (STATIC_CACHE if unique else 0)))
# content-disposition default name
if download:
headers.append(
('Content-Disposition', cls.content_disposition(filename)))
return (status, headers, content)
def test_jpeg(self):
self.assertEqual(
guess_mimetype(contents('jpg')),
'image/jpeg'
)
def test_zip(self):
self.assertEqual(
guess_mimetype(contents('zip')),
'application/zip'
)
def test_odt(self):
self.assertEqual(
guess_mimetype(contents('odt')),
'application/vnd.oasis.opendocument.text'
)
def test_default_mimetype_empty(self):
mimetype = guess_mimetype(b'')
# flectra implementation returns application/octet-stream by default
# if available, python-magic returns application/x-empty
self.assertIn(mimetype,
('application/octet-stream', 'application/x-empty'))
def _binary_record_content(self,
record,
field='datas',
filename=None,
filename_field='name',
default_mimetype='application/octet-stream'):
model = record._name
mimetype = 'mimetype' in record and record.mimetype or False
content = None
filehash = 'checksum' in record and record['checksum'] or False
field_def = record._fields[field]
if field_def.type == 'binary' and field_def.attachment and not field_def.related:
if model != 'ir.attachment':
field_attachment = self.env['ir.attachment'].sudo(
).search_read(domain=[('res_model', '=', model),
('res_id', '=', record.id),
('res_field', '=', field)],
fields=['datas', 'mimetype', 'checksum'],
limit=1)
if field_attachment:
mimetype = field_attachment[0]['mimetype']
content = field_attachment[0]['datas']
filehash = field_attachment[0]['checksum']
else:
mimetype = record['mimetype']
content = record['datas']
filehash = record['checksum']
if not content:
content = record[field] or ''
# filename
default_filename = False
if not filename:
if filename_field in record:
filename = record[filename_field]
if not filename:
default_filename = True
filename = "%s-%s-%s" % (record._name, record.id, field)
if not mimetype:
try:
decoded_content = base64.b64decode(content)
except base64.binascii.Error: # if we could not decode it, no need to pass it down: it would crash elsewhere...
return (404, [], None)
mimetype = guess_mimetype(decoded_content,
default=default_mimetype)
# extension
_, existing_extension = os.path.splitext(filename)
if not existing_extension or default_filename:
extension = mimetypes.guess_extension(mimetype)
if extension:
filename = "%s%s" % (filename, extension)
if not filehash:
filehash = '"%s"' % hashlib.md5(
pycompat.to_text(content).encode('utf-8')).hexdigest()
status = 200 if content else 404
return status, content, filename, mimetype, filehash
def _compute_mimetype(self):
for record in self:
binary = base64.b64decode(record.with_context({}).content or "")
record.res_mimetype = guess_mimetype(
binary, default="application/octet-stream")
def test_mimetype_zip(self):
content = base64.b64decode(ZIP)
mimetype = guess_mimetype(content, default='test')
self.assertEqual(mimetype, 'application/zip')
def test_docx(self):
self.assertEqual(
guess_mimetype(contents('docx')),
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
)
def test_xlsx(self):
self.assertEqual(
guess_mimetype(contents('xlsx')),
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
def test_default_mimetype(self):
mimetype = guess_mimetype(b'', default='test')
# if available, python-magic returns application/x-empty
self.assertIn(mimetype, ('test', 'application/x-empty'))
def test_ods(self):
self.assertEqual(
guess_mimetype(contents('ods')),
'application/vnd.oasis.opendocument.spreadsheet'
)
def test_mimetype_octet_stream(self):
mimetype = guess_mimetype(b'\0')
self.assertEqual(mimetype, 'application/octet-stream')
def test_gif(self):
self.assertEqual(
guess_mimetype(contents('gif')),
'image/gif'
)
def test_mimetype_bmp(self):
content = base64.b64decode(BMP)
mimetype = guess_mimetype(content, default='test')
# mimetype should match image/bmp, image/x-ms-bmp, ...
self.assertRegexpMatches(mimetype, r'image/.*\bbmp')
def test_unknown(self):
self.assertEqual(
guess_mimetype(contents('csv')),
'application/octet-stream'
)
def test_mimetype_jpg(self):
content = base64.b64decode(JPG)
mimetype = guess_mimetype(content, default='test')
self.assertEqual(mimetype, 'image/jpeg')
def get_mimetype(record):
mimetype = mimetypes.guess_type(record.name)[0]
if (not mimetype or mimetype
== 'application/octet-stream') and record.content:
mimetype = guess_mimetype(base64.b64decode(record.content))
return mimetype or 'application/octet-stream'
def test_mimetype_gif(self):
content = base64.b64decode(GIF)
mimetype = guess_mimetype(content, default='test')
self.assertEqual(mimetype, 'image/gif')
def test_doc(self):
self.assertEqual(
guess_mimetype(contents('doc')),
'application/msword'
)
def save_unsplash_url(self, unsplashurls=None, **kwargs):
"""
unsplashurls = {
image_id1: {
url: image_url,
download_url: download_url,
},
image_id2: {
url: image_url,
download_url: download_url,
},
.....
}
"""
def slugify(s):
''' Keeps only alphanumeric characters, hyphens and spaces from a string.
The string will also be truncated to 1024 characters max.
:param s: the string to be filtered
:return: the sanitized string
'''
return "".join([c for c in s
if c.isalnum() or c in list("- ")])[:1024]
if not unsplashurls:
return []
uploads = []
Attachments = request.env['ir.attachment']
query = kwargs.get('query', '')
query = slugify(query)
res_model = kwargs.get('res_model', 'ir.ui.view')
if res_model != 'ir.ui.view' and kwargs.get('res_id'):
res_id = int(kwargs['res_id'])
else:
res_id = None
for key, value in unsplashurls.items():
url = value.get('url')
try:
if not url.startswith('https://images.unsplash.com/'):
logger.exception("ERROR: Unknown Unsplash URL!: " + url)
raise Exception(_("ERROR: Unknown Unsplash URL!"))
req = requests.get(url)
if req.status_code != requests.codes.ok:
continue
# get mime-type of image url because unsplash url dosn't contains mime-types in url
image_base64 = base64.b64encode(req.content)
except requests.exceptions.ConnectionError as e:
logger.exception("Connection Error: " + str(e))
continue
except requests.exceptions.Timeout as e:
logger.exception("Timeout: " + str(e))
continue
image_base64 = tools.image_process(image_base64,
verify_resolution=True)
mimetype = guess_mimetype(base64.b64decode(image_base64))
# append image extension in name
query += mimetypes.guess_extension(mimetype) or ''
# /unsplash/5gR788gfd/lion
url_frags = ['unsplash', key, query]
attachment = Attachments.create({
'name':
'_'.join(url_frags),
'url':
'/' + '/'.join(url_frags),
'mimetype':
mimetype,
'datas':
image_base64,
'public':
res_model == 'ir.ui.view',
'res_id':
res_id,
'res_model':
res_model,
'description':
value.get('description'),
})
attachment.generate_access_token()
uploads.append(attachment._get_media_info())
# Notifies Unsplash from an image download. (API requirement)
self._notify_download(value.get('download_url'))
return uploads
