def _read_file(self, file_type, record, options):
# guess mimetype from file content
mimetype = guess_mimetype(record.file)
(file_extension, handler, req) = FILE_TYPE_DICT.get(mimetype, (None, None, None))
if handler:
try:
return getattr(self, '_read_' + file_extension)(record, options)
except Exception:
_logger.warn("Failed to read file '%s' (transient id %d) using guessed mimetype %s",
record.file_name or '<unknown>', record.id, mimetype)
# try reading with user-provided mimetype
(file_extension, handler, req) = FILE_TYPE_DICT.get(file_type, (None, None, None))
if handler:
try:
return getattr(self, '_read_' + file_extension)(record, options)
except Exception:
_logger.warn("Failed to read file '%s' (transient id %d) using user-provided mimetype %s",
record.file_name or '<unknown>', record.id, file_type)
# fallback on file extensions as mime types can be unreliable (e.g.
# software setting incorrect mime types, or non-installed software
# leading to browser not sending mime types)
if record.file_name:
p, ext = os.path.splitext(record.file_name)
if ext in EXTENSIONS:
try:
return getattr(self, '_read_' + ext[1:])(record, options)
except Exception:
_logger.warn("Failed to read file '%s' (transient id %s) using file extension",
record.file_name, record.id)
if req:
raise ImportError(_("Unable to load \"{extension}\" file: requires Python module \"{modname}\"").format(extension=file_extension, modname=req))
raise ValueError(_("Unsupported file format \"{}\", import only supports CSV, ODS, XLS and XLSX").format(file_type))
def _read_file(self, file_type, record, options):
# guess mimetype from file content
mimetype = guess_mimetype(record.file)
(file_extension, handler, req) = FILE_TYPE_DICT.get(mimetype, (None, None, None))
if handler:
try:
return getattr(self, '_read_' + file_extension)(record, options)
except Exception:
_logger.warn("Failed to read file '%s' (transient id %d) using guessed mimetype %s",
record.file_name or '<unknown>', record.id, mimetype)
# try reading with user-provided mimetype
(file_extension, handler, req) = FILE_TYPE_DICT.get(file_type, (None, None, None))
if handler:
try:
return getattr(self, '_read_' + file_extension)(record, options)
except Exception:
_logger.warn("Failed to read file '%s' (transient id %d) using user-provided mimetype %s",
record.file_name or '<unknown>', record.id, file_type)
# fallback on file extensions as mime types can be unreliable (e.g.
# software setting incorrect mime types, or non-installed software
# leading to browser not sending mime types)
if record.file_name:
p, ext = os.path.splitext(record.file_name)
if ext in EXTENSIONS:
try:
return getattr(self, '_read_' + ext[1:])(record, options)
except Exception:
_logger.warn("Failed to read file '%s' (transient id %s) using file extension",
record.file_name, record.id)
if req:
raise ImportError(_("Unable to load \"{extension}\" file: requires Python module \"{modname}\"").format(extension=file_extension, modname=req))
raise ValueError(_("Unsupported file format \"{}\", import only supports CSV, ODS, XLS and XLSX").format(file_type))
def _compute_mimetype(self, values):
""" compute the mimetype of the given values
:param values : dict of values to create or write an ir_attachment
:return mime : string indicating the mimetype, or application/octet-stream by default
"""
mimetype = 'application/octet-stream'
if values.get('datas_fname'):
mimetype = mimetypes.guess_type(values['datas_fname'])[0]
if values.get('url'):
mimetype = mimetypes.guess_type(values['url'])[0]
if values.get('datas') and (not mimetype or mimetype == 'application/octet-stream'):
mimetype = guess_mimetype(values['datas'].decode('base64'))
return mimetype
def _compute_mimetype(self, values):
""" compute the mimetype of the given values
:param values : dict of values to create or write an ir_attachment
:return mime : string indicating the mimetype, or application/octet-stream by default
"""
mimetype = "application/octet-stream"
if values.get("datas_fname"):
mimetype = mimetypes.guess_type(values["datas_fname"])[0]
if values.get("url"):
mimetype = mimetypes.guess_type(values["url"])[0]
if values.get("datas") and (not mimetype or mimetype == "application/octet-stream"):
mimetype = guess_mimetype(values["datas"].decode("base64"))
return mimetype
def binary_to_form(form, record, fname, value, **req_values):
_value = {
# 'value': '',
# 'raw_value': '',
# 'mimetype': '',
}
if value:
mimetype = guess_mimetype(value.decode('base64'))
_value = {
'value': value,
'raw_value': value,
'mimetype': mimetype,
}
if mimetype.startswith('image/'):
_value['value'] = 'data:{};base64,{}'.format(mimetype, value)
return _value
def test_default_mimetype(self):
mimetype = guess_mimetype('', default='test')
# if available, python-magic returns application/x-empty
self.assertIn(mimetype, ('test', 'application/x-empty'))
def binary_content(self, xmlid=None, model='ir.attachment', id=None, field='datas', unique=False, filename=None, filename_field='datas_fname', download=False, mimetype=None, default_mimetype='application/octet-stream', env=None):
""" Get file, attachment or downloadable content
If the ``xmlid`` and ``id`` parameter is omitted, fetches the default value for the
binary field (via ``default_get``), otherwise fetches the field for
that precise record.
:param str xmlid: xmlid of the record
:param str model: name of the model to fetch the binary from
:param int id: id of the record from which to fetch the binary
:param str field: binary field
:param bool unique: add a max-age for the cache control
:param str filename: choose a filename
:param str filename_field: if not create an filename with model-id-field
:param bool download: apply headers to download the file
:param str mimetype: mintype of the field (for headers)
:param str default_mimetype: default mintype if no mintype found
:param Environment env: by default use request.env
:returns: (status, headers, content)
"""
env = env or request.env
# get object and content
obj = None
if xmlid:
obj = env.ref(xmlid, False)
elif id and model in env.registry:
obj = env[model].browse(int(id))
# obj exists
if not obj or not obj.exists() or field not in obj:
return (404, [], None)
# check read access
try:
last_update = obj['__last_update']
except openerp.exceptions.AccessError:
return (403, [], None)
status, headers, content = None, [], None
# attachment by url check
module_resource_path = None
if model == 'ir.attachment' and obj.type == 'url' and obj.url:
url_match = re.match("^/(\w+)/(.+)$", obj.url)
if url_match:
module = url_match.group(1)
module_path = get_module_path(module)
module_resource_path = get_resource_path(module, url_match.group(2))
if module_path and module_resource_path:
module_path = os.path.join(os.path.normpath(module_path), '') # join ensures the path ends with '/'
module_resource_path = os.path.normpath(module_resource_path)
if module_resource_path.startswith(module_path):
with open(module_resource_path, 'rb') as f:
content = base64.b64encode(f.read())
last_update = str(os.path.getmtime(module_resource_path))
if not module_resource_path:
module_resource_path = obj.url
if not content:
status = 301
content = module_resource_path
else:
content = obj[field] or ''
# filename
if not filename:
if filename_field in obj:
filename = obj[filename_field]
elif module_resource_path:
filename = os.path.basename(module_resource_path)
else:
filename = "%s-%s-%s" % (obj._model._name, obj.id, field)
# mimetype
mimetype = 'mimetype' in obj and obj.mimetype or False
if not mimetype:
if filename:
mimetype = mimetypes.guess_type(filename)[0]
if not mimetype and getattr(env[model]._fields[field], 'attachment', False):
# for binary fields, fetch the ir_attachement for mimetype check
attach_mimetype = env['ir.attachment'].search_read(domain=[('res_model', '=', model), ('res_id', '=', id), ('res_field', '=', field)], fields=['mimetype'], limit=1)
mimetype = attach_mimetype and attach_mimetype[0]['mimetype']
if not mimetype:
mimetype = guess_mimetype(base64.b64decode(content), default=default_mimetype)
headers += [('Content-Type', mimetype), ('X-Content-Type-Options', 'nosniff')]
# cache
etag = hasattr(request, 'httprequest') and request.httprequest.headers.get('If-None-Match')
retag = '"%s"' % hashlib.md5(last_update).hexdigest()
status = status or (304 if etag == retag else 200)
headers.append(('ETag', retag))
headers.append(('Cache-Control', 'max-age=%s' % (STATIC_CACHE if unique else 0)))
# content-disposition default name
if download:
headers.append(('Content-Disposition', self.content_disposition(filename)))
return (status, headers, content)
def test_jpeg(self):
self.assertEqual(
guess_mimetype(contents('jpg')),
'image/jpeg'
)
def test_zip(self):
self.assertEqual(
guess_mimetype(contents('zip')),
'application/zip'
)
def binary_content(self, xmlid=None, model='ir.attachment', id=None, field='datas', unique=False, filename=None, filename_field='datas_fname', download=False, mimetype=None, default_mimetype='application/octet-stream', env=None):
""" Get file, attachment or downloadable content
If the ``xmlid`` and ``id`` parameter is omitted, fetches the default value for the
binary field (via ``default_get``), otherwise fetches the field for
that precise record.
:param str xmlid: xmlid of the record
:param str model: name of the model to fetch the binary from
:param int id: id of the record from which to fetch the binary
:param str field: binary field
:param bool unique: add a max-age for the cache control
:param str filename: choose a filename
:param str filename_field: if not create an filename with model-id-field
:param bool download: apply headers to download the file
:param str mimetype: mintype of the field (for headers)
:param str default_mimetype: default mintype if no mintype found
:param Environment env: by default use request.env
:returns: (status, headers, content)
"""
env = env or request.env
# get object and content
obj = None
if xmlid:
obj = env.ref(xmlid, False)
elif id and model in env.registry:
obj = env[model].browse(int(id))
# obj exists
if not obj or not obj.exists() or field not in obj:
return (404, [], None)
# check read access
try:
last_update = obj['__last_update']
except openerp.exceptions.AccessError:
return (403, [], None)
status, headers, content = None, [], None
# attachment by url check
module_resource_path = None
if model == 'ir.attachment' and obj.type == 'url' and obj.url:
url_match = re.match("^/(\w+)/(.+)$", obj.url)
if url_match:
module = url_match.group(1)
module_path = get_module_path(module)
module_resource_path = get_resource_path(module, url_match.group(2))
if module_path and module_resource_path:
module_path = os.path.join(os.path.normpath(module_path), '') # join ensures the path ends with '/'
module_resource_path = os.path.normpath(module_resource_path)
if module_resource_path.startswith(module_path):
with open(module_resource_path, 'r') as f:
content = base64.b64encode(f.read())
last_update = str(os.path.getmtime(module_resource_path))
if not module_resource_path:
module_resource_path = obj.url
if not content:
status = 301
content = module_resource_path
else:
content = obj[field] or ''
# filename
if not filename:
if filename_field in obj:
filename = obj[filename_field]
elif module_resource_path:
filename = os.path.basename(module_resource_path)
else:
filename = "%s-%s-%s" % (obj._model._name, obj.id, field)
# mimetype
mimetype = 'mimetype' in obj and obj.mimetype or False
if not mimetype:
if filename:
mimetype = mimetypes.guess_type(filename)[0]
if not mimetype and getattr(env[model]._fields[field], 'attachment', False):
# for binary fields, fetch the ir_attachement for mimetype check
attach_mimetype = env['ir.attachment'].search_read(domain=[('res_model', '=', model), ('res_id', '=', id), ('res_field', '=', field)], fields=['mimetype'], limit=1)
mimetype = attach_mimetype and attach_mimetype[0]['mimetype']
if not mimetype:
mimetype = guess_mimetype(base64.b64decode(content), default=default_mimetype)
headers += [('Content-Type', mimetype), ('X-Content-Type-Options', 'nosniff')]
# cache
etag = hasattr(request, 'httprequest') and request.httprequest.headers.get('If-None-Match')
retag = '"%s"' % hashlib.md5(last_update).hexdigest()
status = status or (304 if etag == retag else 200)
headers.append(('ETag', retag))
headers.append(('Cache-Control', 'max-age=%s' % (STATIC_CACHE if unique else 0)))
# content-disposition default name
if download:
headers.append(('Content-Disposition', self.content_disposition(filename)))
return (status, headers, content)
def test_mimetype_gif(self):
content = base64.b64decode(GIF)
mimetype = guess_mimetype(content, default='test')
self.assertEqual(mimetype, 'image/gif')
def test_doc(self):
self.assertEqual(guess_mimetype(contents('doc')), 'application/msword')
def test_mimetype_bmp(self):
content = base64.b64decode(BMP)
mimetype = guess_mimetype(content, default='test')
self.assertEqual(mimetype, 'image/bmp')
def test_mimetype_jpg(self):
content = base64.b64decode(JPG)
mimetype = guess_mimetype(content, default='test')
self.assertEqual(mimetype, 'image/jpeg')
def test_default_mimetype(self):
mimetype = guess_mimetype('', default='test')
self.assertEqual(mimetype, 'test')
def test_default_mimetype_empty(self):
mimetype = guess_mimetype('')
self.assertEqual(mimetype, 'application/octet-stream')
def _parse_file(self, data_file):
""" Each module adding a file support must extends this method. It processes the file if it can, returns super otherwise, resulting in a chain of responsability.
This method parses the given file and returns the data required by the bank statement import process, as specified below.
rtype: triplet (if a value can't be retrieved, use None)
- currency code: string (e.g: 'EUR')
The ISO 4217 currency code, case insensitive
- account number: string (e.g: 'BE1234567890')
The number of the bank account which the statement belongs to
- bank statements data: list of dict containing (optional items marked by o) :
- 'name': string (e.g: '000000123')
- 'date': date (e.g: 2013-06-26)
-o 'balance_start': float (e.g: 8368.56)
-o 'balance_end_real': float (e.g: 8888.88)
- 'transactions': list of dict containing :
- 'name': string (e.g: 'KBC-INVESTERINGSKREDIET 787-5562831-01')
- 'date': date
- 'amount': float
- 'unique_import_id': string. # Ensure transactions can be imported only once (if the import format provides unique transaction ids)
-o 'account_number': string
Will be used to find/create the res.partner.bank in odoo
-o 'note': string
-o 'partner_name': string
-o 'ref': string
"""
#Validar Formato
# guess mimetype from file content
mimetype = guess_mimetype(data_file)
_logger.debug('JC mimetype %s', mimetype)
#Leer Archivo
Import = self.env['base_import.import']
id = Import.create({
'res_model':
'campus_bancos.formato_banco_loja',
'file':
data_file,
'file_type':
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
})
#raise UserError(_('El arhivo no tiene formato correcto'))
#obtener los campos del modelo
fields = Import.get_fields('campus_bancos.formato_banco_loja')
_logger.debug('JC fields %s', fields)
_logger.debug('JC _parse_file self %s', self)
#_logger.debug('JC data_file %s',data_file)
_logger.debug('JC id %s', id)
#record = Import.browse(id)
#_logger.debug('record %s',record)
#error: can't adapt type 'base_import.import'
#rows = Import._read_file(record.file_type, record, {'headers': True})
#_read_xls(self, record, options)
options = {'headers': True}
rows_to_import = self._read_xls(data_file, options)
headers, matches = Import._match_headers(rows_to_import, fields,
options)
_logger.debug('headers %s', headers)
_logger.debug('matches %s', matches)
if options.get('headers'):
rows_to_import = itertools.islice(rows_to_import, 1, None)
#TODO: Ver la forma de mapear los indices
#{0: ['fecha'], 1: ['transaccion'], 2: ['rubro'], 3: ['concepto'], 4: ['referencia'],
#5: ['oficina'], 6: ['debitos'], 7: ['creditos'], 8: ['saldo'], 9: ['id']}
#[u'2016/12/01 00:00:00', u'Saldo Inicial', u'Saldo', u'', u'12', u'', u'0', u'0', u'9656.4', u''], [u'2016/12/05 15:58:36', u'NOTA DE DEBITO', u'RETIRO EN CAJERO AUTOM\xc1TICO', u'Universidad Internacional', u'5267', u'OFICINA MATRIZ', u'60', u'0', u'9596.4', u''],
#TODO: saltar la fila Saldo Inicial. 0
transactions = []
vals_line = {}
total = 0
vals_bank_statement = {}
#TODO: Utilizar itertools, para mejor rendimiento
#for line in itertools.imap(rows_to_import, len(rows_to_import)):
for line in rows_to_import:
vals_line['date'] = dateutil.parser.parse(line[0],
fuzzy=True).date()
vals_line['name'] = line[1] + ' : ' + line[2] + ' : ' + line[3]
vals_line['ref'] = line[4]
vals_line['note'] = line[1] + ' : ' + line[2]
#TODO: Separador de decimales, y decenas
if line[6] and float(line[6].replace(',', '')) != 0:
vals_line['amount'] = float(line[6].replace(',', '')) * -1
elif line[7] and float(line[7].replace(',', '')) != 0:
vals_line['amount'] = float(line[7].replace(',', ''))
total += vals_line['amount']
transactions.append(vals_line)
vals_line = {}
_logger.debug('JC total %s', total)
_logger.debug('JC transactions %s', transactions)
# initialize a new statement
#statement = {
# 'name' : account + '-' + st.statement + '-' + st.information,
# 'date' : st.end_balance.date,
# 'balance_start' : st.start_balance.amount,
# 'balance_end_real' : st.end_balance.amount,
# 'transactions' : []
#}
vals_bank_statement.update({
'balance_end_real': total,
'transactions': transactions
})
#return currency, account, statements
return None, None, [vals_bank_statement]
def test_jpeg(self):
self.assertEqual(guess_mimetype(contents('jpg')), 'image/jpeg')
def test_doc(self):
self.assertEqual(
guess_mimetype(contents('doc')),
'application/msword'
)
def test_unknown(self):
self.assertEqual(guess_mimetype(contents('csv')),
'application/octet-stream')
def test_docx(self):
self.assertEqual(
guess_mimetype(contents('docx')),
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
)
def test_mimetype_octet_stream(self):
mimetype = guess_mimetype('\0')
self.assertEqual(mimetype, 'application/octet-stream')
def test_odt(self):
self.assertEqual(
guess_mimetype(contents('odt')),
'application/vnd.oasis.opendocument.text'
)
def test_xls(self):
self.assertEqual(guess_mimetype(contents('xls')),
'application/vnd.ms-excel')
def test_mimetype_bmp(self):
content = base64.b64decode(BMP)
mimetype = guess_mimetype(content, default='test')
# mimetype should match image/bmp, image/x-ms-bmp, ...
self.assertRegexpMatches(mimetype, r'image/.*\bbmp')
def test_xls(self):
self.assertEqual(
guess_mimetype(contents('xls')),
'application/vnd.ms-excel'
)
def test_docx(self):
self.assertEqual(
guess_mimetype(contents('docx')),
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
)
def test_xlsx(self):
self.assertEqual(
guess_mimetype(contents('xlsx')),
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
def test_xlsx(self):
self.assertEqual(
guess_mimetype(contents('xlsx')),
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
def test_ods(self):
self.assertEqual(
guess_mimetype(contents('ods')),
'application/vnd.oasis.opendocument.spreadsheet'
)
def test_odt(self):
self.assertEqual(guess_mimetype(contents('odt')),
'application/vnd.oasis.opendocument.text')
def test_gif(self):
self.assertEqual(
guess_mimetype(contents('gif')),
'image/gif'
)
def test_ods(self):
self.assertEqual(guess_mimetype(contents('ods')),
'application/vnd.oasis.opendocument.spreadsheet')
def test_unknown(self):
self.assertEqual(
guess_mimetype(contents('csv')),
'application/octet-stream'
)
def test_zip(self):
self.assertEqual(guess_mimetype(contents('zip')), 'application/zip')
def test_gif(self):
self.assertEqual(guess_mimetype(contents('gif')), 'image/gif')
def test_default_mimetype_empty(self):
mimetype = guess_mimetype('')
# odoo implementation returns application/octet-stream by default
# if available, python-magic returns application/x-empty
self.assertIn(mimetype,
('application/octet-stream', 'application/x-empty'))
