def _read_file(self, file_type, record, options): # guess mimetype from file content mimetype = guess_mimetype(record.file) (file_extension, handler, req) = FILE_TYPE_DICT.get(mimetype, (None, None, None)) if handler: try: return getattr(self, '_read_' + file_extension)(record, options) except Exception: _logger.warn("Failed to read file '%s' (transient id %d) using guessed mimetype %s", record.file_name or '<unknown>', record.id, mimetype) # try reading with user-provided mimetype (file_extension, handler, req) = FILE_TYPE_DICT.get(file_type, (None, None, None)) if handler: try: return getattr(self, '_read_' + file_extension)(record, options) except Exception: _logger.warn("Failed to read file '%s' (transient id %d) using user-provided mimetype %s", record.file_name or '<unknown>', record.id, file_type) # fallback on file extensions as mime types can be unreliable (e.g. # software setting incorrect mime types, or non-installed software # leading to browser not sending mime types) if record.file_name: p, ext = os.path.splitext(record.file_name) if ext in EXTENSIONS: try: return getattr(self, '_read_' + ext[1:])(record, options) except Exception: _logger.warn("Failed to read file '%s' (transient id %s) using file extension", record.file_name, record.id) if req: raise ImportError(_("Unable to load \"{extension}\" file: requires Python module \"{modname}\"").format(extension=file_extension, modname=req)) raise ValueError(_("Unsupported file format \"{}\", import only supports CSV, ODS, XLS and XLSX").format(file_type))
def _compute_mimetype(self, values): """ compute the mimetype of the given values :param values : dict of values to create or write an ir_attachment :return mime : string indicating the mimetype, or application/octet-stream by default """ mimetype = 'application/octet-stream' if values.get('datas_fname'): mimetype = mimetypes.guess_type(values['datas_fname'])[0] if values.get('url'): mimetype = mimetypes.guess_type(values['url'])[0] if values.get('datas') and (not mimetype or mimetype == 'application/octet-stream'): mimetype = guess_mimetype(values['datas'].decode('base64')) return mimetype
def _compute_mimetype(self, values): """ compute the mimetype of the given values :param values : dict of values to create or write an ir_attachment :return mime : string indicating the mimetype, or application/octet-stream by default """ mimetype = "application/octet-stream" if values.get("datas_fname"): mimetype = mimetypes.guess_type(values["datas_fname"])[0] if values.get("url"): mimetype = mimetypes.guess_type(values["url"])[0] if values.get("datas") and (not mimetype or mimetype == "application/octet-stream"): mimetype = guess_mimetype(values["datas"].decode("base64")) return mimetype
def binary_to_form(form, record, fname, value, **req_values): _value = { # 'value': '', # 'raw_value': '', # 'mimetype': '', } if value: mimetype = guess_mimetype(value.decode('base64')) _value = { 'value': value, 'raw_value': value, 'mimetype': mimetype, } if mimetype.startswith('image/'): _value['value'] = 'data:{};base64,{}'.format(mimetype, value) return _value
def test_default_mimetype(self): mimetype = guess_mimetype('', default='test') # if available, python-magic returns application/x-empty self.assertIn(mimetype, ('test', 'application/x-empty'))
def binary_content(self, xmlid=None, model='ir.attachment', id=None, field='datas', unique=False, filename=None, filename_field='datas_fname', download=False, mimetype=None, default_mimetype='application/octet-stream', env=None): """ Get file, attachment or downloadable content If the ``xmlid`` and ``id`` parameter is omitted, fetches the default value for the binary field (via ``default_get``), otherwise fetches the field for that precise record. :param str xmlid: xmlid of the record :param str model: name of the model to fetch the binary from :param int id: id of the record from which to fetch the binary :param str field: binary field :param bool unique: add a max-age for the cache control :param str filename: choose a filename :param str filename_field: if not create an filename with model-id-field :param bool download: apply headers to download the file :param str mimetype: mintype of the field (for headers) :param str default_mimetype: default mintype if no mintype found :param Environment env: by default use request.env :returns: (status, headers, content) """ env = env or request.env # get object and content obj = None if xmlid: obj = env.ref(xmlid, False) elif id and model in env.registry: obj = env[model].browse(int(id)) # obj exists if not obj or not obj.exists() or field not in obj: return (404, [], None) # check read access try: last_update = obj['__last_update'] except openerp.exceptions.AccessError: return (403, [], None) status, headers, content = None, [], None # attachment by url check module_resource_path = None if model == 'ir.attachment' and obj.type == 'url' and obj.url: url_match = re.match("^/(\w+)/(.+)$", obj.url) if url_match: module = url_match.group(1) module_path = get_module_path(module) module_resource_path = get_resource_path(module, url_match.group(2)) if module_path and module_resource_path: module_path = os.path.join(os.path.normpath(module_path), '') # join ensures the path ends with '/' module_resource_path = os.path.normpath(module_resource_path) if module_resource_path.startswith(module_path): with open(module_resource_path, 'rb') as f: content = base64.b64encode(f.read()) last_update = str(os.path.getmtime(module_resource_path)) if not module_resource_path: module_resource_path = obj.url if not content: status = 301 content = module_resource_path else: content = obj[field] or '' # filename if not filename: if filename_field in obj: filename = obj[filename_field] elif module_resource_path: filename = os.path.basename(module_resource_path) else: filename = "%s-%s-%s" % (obj._model._name, obj.id, field) # mimetype mimetype = 'mimetype' in obj and obj.mimetype or False if not mimetype: if filename: mimetype = mimetypes.guess_type(filename)[0] if not mimetype and getattr(env[model]._fields[field], 'attachment', False): # for binary fields, fetch the ir_attachement for mimetype check attach_mimetype = env['ir.attachment'].search_read(domain=[('res_model', '=', model), ('res_id', '=', id), ('res_field', '=', field)], fields=['mimetype'], limit=1) mimetype = attach_mimetype and attach_mimetype[0]['mimetype'] if not mimetype: mimetype = guess_mimetype(base64.b64decode(content), default=default_mimetype) headers += [('Content-Type', mimetype), ('X-Content-Type-Options', 'nosniff')] # cache etag = hasattr(request, 'httprequest') and request.httprequest.headers.get('If-None-Match') retag = '"%s"' % hashlib.md5(last_update).hexdigest() status = status or (304 if etag == retag else 200) headers.append(('ETag', retag)) headers.append(('Cache-Control', 'max-age=%s' % (STATIC_CACHE if unique else 0))) # content-disposition default name if download: headers.append(('Content-Disposition', self.content_disposition(filename))) return (status, headers, content)
def test_jpeg(self): self.assertEqual( guess_mimetype(contents('jpg')), 'image/jpeg' )
def test_zip(self): self.assertEqual( guess_mimetype(contents('zip')), 'application/zip' )
def binary_content(self, xmlid=None, model='ir.attachment', id=None, field='datas', unique=False, filename=None, filename_field='datas_fname', download=False, mimetype=None, default_mimetype='application/octet-stream', env=None): """ Get file, attachment or downloadable content If the ``xmlid`` and ``id`` parameter is omitted, fetches the default value for the binary field (via ``default_get``), otherwise fetches the field for that precise record. :param str xmlid: xmlid of the record :param str model: name of the model to fetch the binary from :param int id: id of the record from which to fetch the binary :param str field: binary field :param bool unique: add a max-age for the cache control :param str filename: choose a filename :param str filename_field: if not create an filename with model-id-field :param bool download: apply headers to download the file :param str mimetype: mintype of the field (for headers) :param str default_mimetype: default mintype if no mintype found :param Environment env: by default use request.env :returns: (status, headers, content) """ env = env or request.env # get object and content obj = None if xmlid: obj = env.ref(xmlid, False) elif id and model in env.registry: obj = env[model].browse(int(id)) # obj exists if not obj or not obj.exists() or field not in obj: return (404, [], None) # check read access try: last_update = obj['__last_update'] except openerp.exceptions.AccessError: return (403, [], None) status, headers, content = None, [], None # attachment by url check module_resource_path = None if model == 'ir.attachment' and obj.type == 'url' and obj.url: url_match = re.match("^/(\w+)/(.+)$", obj.url) if url_match: module = url_match.group(1) module_path = get_module_path(module) module_resource_path = get_resource_path(module, url_match.group(2)) if module_path and module_resource_path: module_path = os.path.join(os.path.normpath(module_path), '') # join ensures the path ends with '/' module_resource_path = os.path.normpath(module_resource_path) if module_resource_path.startswith(module_path): with open(module_resource_path, 'r') as f: content = base64.b64encode(f.read()) last_update = str(os.path.getmtime(module_resource_path)) if not module_resource_path: module_resource_path = obj.url if not content: status = 301 content = module_resource_path else: content = obj[field] or '' # filename if not filename: if filename_field in obj: filename = obj[filename_field] elif module_resource_path: filename = os.path.basename(module_resource_path) else: filename = "%s-%s-%s" % (obj._model._name, obj.id, field) # mimetype mimetype = 'mimetype' in obj and obj.mimetype or False if not mimetype: if filename: mimetype = mimetypes.guess_type(filename)[0] if not mimetype and getattr(env[model]._fields[field], 'attachment', False): # for binary fields, fetch the ir_attachement for mimetype check attach_mimetype = env['ir.attachment'].search_read(domain=[('res_model', '=', model), ('res_id', '=', id), ('res_field', '=', field)], fields=['mimetype'], limit=1) mimetype = attach_mimetype and attach_mimetype[0]['mimetype'] if not mimetype: mimetype = guess_mimetype(base64.b64decode(content), default=default_mimetype) headers += [('Content-Type', mimetype), ('X-Content-Type-Options', 'nosniff')] # cache etag = hasattr(request, 'httprequest') and request.httprequest.headers.get('If-None-Match') retag = '"%s"' % hashlib.md5(last_update).hexdigest() status = status or (304 if etag == retag else 200) headers.append(('ETag', retag)) headers.append(('Cache-Control', 'max-age=%s' % (STATIC_CACHE if unique else 0))) # content-disposition default name if download: headers.append(('Content-Disposition', self.content_disposition(filename))) return (status, headers, content)
def test_mimetype_gif(self): content = base64.b64decode(GIF) mimetype = guess_mimetype(content, default='test') self.assertEqual(mimetype, 'image/gif')
def test_doc(self): self.assertEqual(guess_mimetype(contents('doc')), 'application/msword')
def test_mimetype_bmp(self): content = base64.b64decode(BMP) mimetype = guess_mimetype(content, default='test') self.assertEqual(mimetype, 'image/bmp')
def test_mimetype_jpg(self): content = base64.b64decode(JPG) mimetype = guess_mimetype(content, default='test') self.assertEqual(mimetype, 'image/jpeg')
def test_default_mimetype(self): mimetype = guess_mimetype('', default='test') self.assertEqual(mimetype, 'test')
def test_default_mimetype_empty(self): mimetype = guess_mimetype('') self.assertEqual(mimetype, 'application/octet-stream')
def _parse_file(self, data_file): """ Each module adding a file support must extends this method. It processes the file if it can, returns super otherwise, resulting in a chain of responsability. This method parses the given file and returns the data required by the bank statement import process, as specified below. rtype: triplet (if a value can't be retrieved, use None) - currency code: string (e.g: 'EUR') The ISO 4217 currency code, case insensitive - account number: string (e.g: 'BE1234567890') The number of the bank account which the statement belongs to - bank statements data: list of dict containing (optional items marked by o) : - 'name': string (e.g: '000000123') - 'date': date (e.g: 2013-06-26) -o 'balance_start': float (e.g: 8368.56) -o 'balance_end_real': float (e.g: 8888.88) - 'transactions': list of dict containing : - 'name': string (e.g: 'KBC-INVESTERINGSKREDIET 787-5562831-01') - 'date': date - 'amount': float - 'unique_import_id': string. # Ensure transactions can be imported only once (if the import format provides unique transaction ids) -o 'account_number': string Will be used to find/create the res.partner.bank in odoo -o 'note': string -o 'partner_name': string -o 'ref': string """ #Validar Formato # guess mimetype from file content mimetype = guess_mimetype(data_file) _logger.debug('JC mimetype %s', mimetype) #Leer Archivo Import = self.env['base_import.import'] id = Import.create({ 'res_model': 'campus_bancos.formato_banco_loja', 'file': data_file, 'file_type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' }) #raise UserError(_('El arhivo no tiene formato correcto')) #obtener los campos del modelo fields = Import.get_fields('campus_bancos.formato_banco_loja') _logger.debug('JC fields %s', fields) _logger.debug('JC _parse_file self %s', self) #_logger.debug('JC data_file %s',data_file) _logger.debug('JC id %s', id) #record = Import.browse(id) #_logger.debug('record %s',record) #error: can't adapt type 'base_import.import' #rows = Import._read_file(record.file_type, record, {'headers': True}) #_read_xls(self, record, options) options = {'headers': True} rows_to_import = self._read_xls(data_file, options) headers, matches = Import._match_headers(rows_to_import, fields, options) _logger.debug('headers %s', headers) _logger.debug('matches %s', matches) if options.get('headers'): rows_to_import = itertools.islice(rows_to_import, 1, None) #TODO: Ver la forma de mapear los indices #{0: ['fecha'], 1: ['transaccion'], 2: ['rubro'], 3: ['concepto'], 4: ['referencia'], #5: ['oficina'], 6: ['debitos'], 7: ['creditos'], 8: ['saldo'], 9: ['id']} #[u'2016/12/01 00:00:00', u'Saldo Inicial', u'Saldo', u'', u'12', u'', u'0', u'0', u'9656.4', u''], [u'2016/12/05 15:58:36', u'NOTA DE DEBITO', u'RETIRO EN CAJERO AUTOM\xc1TICO', u'Universidad Internacional', u'5267', u'OFICINA MATRIZ', u'60', u'0', u'9596.4', u''], #TODO: saltar la fila Saldo Inicial. 0 transactions = [] vals_line = {} total = 0 vals_bank_statement = {} #TODO: Utilizar itertools, para mejor rendimiento #for line in itertools.imap(rows_to_import, len(rows_to_import)): for line in rows_to_import: vals_line['date'] = dateutil.parser.parse(line[0], fuzzy=True).date() vals_line['name'] = line[1] + ' : ' + line[2] + ' : ' + line[3] vals_line['ref'] = line[4] vals_line['note'] = line[1] + ' : ' + line[2] #TODO: Separador de decimales, y decenas if line[6] and float(line[6].replace(',', '')) != 0: vals_line['amount'] = float(line[6].replace(',', '')) * -1 elif line[7] and float(line[7].replace(',', '')) != 0: vals_line['amount'] = float(line[7].replace(',', '')) total += vals_line['amount'] transactions.append(vals_line) vals_line = {} _logger.debug('JC total %s', total) _logger.debug('JC transactions %s', transactions) # initialize a new statement #statement = { # 'name' : account + '-' + st.statement + '-' + st.information, # 'date' : st.end_balance.date, # 'balance_start' : st.start_balance.amount, # 'balance_end_real' : st.end_balance.amount, # 'transactions' : [] #} vals_bank_statement.update({ 'balance_end_real': total, 'transactions': transactions }) #return currency, account, statements return None, None, [vals_bank_statement]
def test_jpeg(self): self.assertEqual(guess_mimetype(contents('jpg')), 'image/jpeg')
def test_doc(self): self.assertEqual( guess_mimetype(contents('doc')), 'application/msword' )
def test_unknown(self): self.assertEqual(guess_mimetype(contents('csv')), 'application/octet-stream')
def test_docx(self): self.assertEqual( guess_mimetype(contents('docx')), 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' )
def test_mimetype_octet_stream(self): mimetype = guess_mimetype('\0') self.assertEqual(mimetype, 'application/octet-stream')
def test_odt(self): self.assertEqual( guess_mimetype(contents('odt')), 'application/vnd.oasis.opendocument.text' )
def test_xls(self): self.assertEqual(guess_mimetype(contents('xls')), 'application/vnd.ms-excel')
def test_mimetype_bmp(self): content = base64.b64decode(BMP) mimetype = guess_mimetype(content, default='test') # mimetype should match image/bmp, image/x-ms-bmp, ... self.assertRegexpMatches(mimetype, r'image/.*\bbmp')
def test_xls(self): self.assertEqual( guess_mimetype(contents('xls')), 'application/vnd.ms-excel' )
def test_xlsx(self): self.assertEqual( guess_mimetype(contents('xlsx')), 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' )
def test_ods(self): self.assertEqual( guess_mimetype(contents('ods')), 'application/vnd.oasis.opendocument.spreadsheet' )
def test_odt(self): self.assertEqual(guess_mimetype(contents('odt')), 'application/vnd.oasis.opendocument.text')
def test_gif(self): self.assertEqual( guess_mimetype(contents('gif')), 'image/gif' )
def test_ods(self): self.assertEqual(guess_mimetype(contents('ods')), 'application/vnd.oasis.opendocument.spreadsheet')
def test_unknown(self): self.assertEqual( guess_mimetype(contents('csv')), 'application/octet-stream' )
def test_zip(self): self.assertEqual(guess_mimetype(contents('zip')), 'application/zip')
def test_gif(self): self.assertEqual(guess_mimetype(contents('gif')), 'image/gif')
def test_default_mimetype_empty(self): mimetype = guess_mimetype('') # odoo implementation returns application/octet-stream by default # if available, python-magic returns application/x-empty self.assertIn(mimetype, ('application/octet-stream', 'application/x-empty'))