def _makefile(self, rows): f = io.BytesIO() writer = pycompat.csv_writer(f, quoting=1) writer.writerow(['name', 'counter']) for i in range(rows): writer.writerow(['n_%d' % i, str(i)]) return f.getvalue()
def test_big_attachments(self): """ Ensure big fields (e.g. b64-encoded image data) can be imported and we're not hitting limits of the default CSV parser config """ from PIL import Image im = Image.new('RGB', (1920, 1080)) fout = io.BytesIO() writer = pycompat.csv_writer(fout, dialect=None) writer.writerows([ [u'name', u'db_datas'], [u'foo', base64.b64encode(im.tobytes()).decode('ascii')] ]) import_wizard = self.env['base_import.import'].create({ 'res_model': 'ir.attachment', 'file': fout.getvalue(), 'file_type': 'text/csv' }) results = import_wizard.do( ['name', 'db_datas'], [], {'headers': True, 'separator': ',', 'quoting': '"'}) self.assertFalse(results['messages'], "results should be empty on successful import")
def fiscal_pos_map_to_csv(self): writer = pycompat.csv_writer( open( 'account.fiscal.' 'position.tax.template-%s.csv' % self.suffix, 'wb')) fiscal_pos_map_iterator = self.iter_fiscal_pos_map() keys = next(fiscal_pos_map_iterator) writer.writerow(keys) for row in fiscal_pos_map_iterator: writer.writerow([pycompat.to_text(s) for s in row.values()])
def test_limit_on_lines(self): """ The limit option should be a limit on the number of *lines* imported at at time, not the number of *records*. This is relevant when it comes to embedded o2m. A big question is whether we want to round up or down (if the limit brings us inside a record). Rounding up (aka finishing up the record we're currently parsing) seems like a better idea: * if the first record has so many sub-lines it hits the limit we still want to import it (it's probably extremely rare but it can happen) * if we have one line per record, we probably want to import <limit> records not <limit-1>, but if we stop in the middle of the "current record" we'd always ignore the last record (I think) """ f = io.BytesIO() writer = pycompat.csv_writer(f, quoting=1) writer.writerow(['name', 'value/value']) for record in range(10): writer.writerow(['record_%d' % record, '0']) for row in range(1, 10): writer.writerow(['', str(row)]) import_wizard = self.env['base_import.import'].create({ 'res_model': 'base_import.tests.models.o2m', 'file_type': 'text/csv', 'file_name': 'things.csv', 'file': f.getvalue(), }) opts = {'quoting': '"', 'separator': ',', 'headers': True} preview = import_wizard.parse_preview({**opts, 'limit': 15}) self.assertIs(preview['batch'], True) results = import_wizard.do( ['name', 'value/value'], [], {**opts, 'limit': 5} ) self.assertFalse(results['messages']) self.assertEqual(len(results['ids']), 1, "should have imported the first record in full, got %s" % results['ids']) self.assertEqual(results['nextrow'], 10) results = import_wizard.do( ['name', 'value/value'], [], {**opts, 'limit': 15} ) self.assertFalse(results['messages']) self.assertEqual(len(results['ids']), 2, "should have importe the first two records, got %s" % results['ids']) self.assertEqual(results['nextrow'], 20)
def taxes_to_csv(self): writer = pycompat.csv_writer( open('account.tax.template-%s.csv' % self.suffix, 'wb')) taxes_iterator = self.iter_taxes() keys = next(taxes_iterator) writer.writerow(keys[3:] + ['sequence']) seq = 100 for row in sorted(taxes_iterator, key=lambda r: r['description']): if not _is_true(row['active']): continue seq += 1 if row['parent_id:id']: cur_seq = seq + 1000 else: cur_seq = seq writer.writerow( [pycompat.to_text(v) for v in list(row.values())[3:]] + [cur_seq])
def test_newline_import(self): """ Ensure importing keep newlines """ output = io.BytesIO() writer = pycompat.csv_writer(output, quoting=1) data_row = [u"\tfoo\n\tbar", u" \"hello\" \n\n 'world' "] writer.writerow([u"name", u"Some Value"]) writer.writerow(data_row) import_wizard = self.env['base_import.import'].create({ 'res_model': 'base_import.tests.models.preview', 'file': output.getvalue(), 'file_type': 'text/csv', }) data, _ = import_wizard._convert_import_data( ['name', 'somevalue'], {'quoting': '"', 'separator': ',', 'headers': True} ) self.assertItemsEqual(data, [data_row])
def tax_codes_to_csv(self): writer = pycompat.csv_writer( open('account.tax.code.template-%s.csv' % self.suffix, 'wb')) tax_codes_iterator = self.iter_tax_codes() keys = next(tax_codes_iterator) writer.writerow(keys) # write structure tax codes tax_codes = {} # code: id for row in tax_codes_iterator: tax_code = row['code'] if tax_code in tax_codes: raise RuntimeError('duplicate tax code %s' % tax_code) tax_codes[tax_code] = row['id'] writer.writerow([pycompat.to_text(v) for v in row.values()]) # read taxes and add leaf tax codes new_tax_codes = {} # id: parent_code def add_new_tax_code(tax_code_id, new_name, new_parent_code): if not tax_code_id: return name, parent_code = new_tax_codes.get(tax_code_id, (None, None)) if parent_code and parent_code != new_parent_code: raise RuntimeError('tax code "%s" already exist with ' 'parent %s while trying to add it with ' 'parent %s' % (tax_code_id, parent_code, new_parent_code)) else: new_tax_codes[tax_code_id] = (new_name, new_parent_code) taxes_iterator = self.iter_taxes() next(taxes_iterator) for row in taxes_iterator: if not _is_true(row['active']): continue if row['child_depend'] and row['amount'] != 1: raise RuntimeError('amount must be one if child_depend ' 'for %s' % row['id']) # base parent base_code = row['BASE_CODE'] if not base_code or base_code == '/': base_code = 'NA' if base_code not in tax_codes: raise RuntimeError('undefined tax code %s' % base_code) if base_code != 'NA': if row['child_depend']: raise RuntimeError('base code specified ' 'with child_depend for %s' % row['id']) if not row['child_depend']: # ... in lux, we have the same code for invoice and refund if base_code != 'NA': assert row[ 'base_code_id:id'], 'missing base_code_id for %s' % row[ 'id'] assert row['ref_base_code_id:id'] == row['base_code_id:id'] add_new_tax_code(row['base_code_id:id'], 'Base - ' + row['name'], base_code) # tax parent tax_code = row['TAX_CODE'] if not tax_code or tax_code == '/': tax_code = 'NA' if tax_code not in tax_codes: raise RuntimeError('undefined tax code %s' % tax_code) if tax_code == 'NA': if row['amount'] and not row['child_depend']: raise RuntimeError('TAX_CODE not specified ' 'for non-zero tax %s' % row['id']) if row['tax_code_id:id']: raise RuntimeError('tax_code_id specified ' 'for tax %s' % row['id']) else: if row['child_depend']: raise RuntimeError('TAX_CODE specified ' 'with child_depend for %s' % row['id']) if not row['amount']: raise RuntimeError('TAX_CODE specified ' 'for zero tax %s' % row['id']) if not row['tax_code_id:id']: raise RuntimeError('tax_code_id not specified ' 'for tax %s' % row['id']) if not row['child_depend'] and row['amount']: # ... in lux, we have the same code for invoice and refund assert row[ 'tax_code_id:id'], 'missing tax_code_id for %s' % row['id'] assert row['ref_tax_code_id:id'] == row['tax_code_id:id'] add_new_tax_code(row['tax_code_id:id'], 'Taxe - ' + row['name'], tax_code) for tax_code_id in sorted(new_tax_codes): name, parent_code = new_tax_codes[tax_code_id] writer.writerow([ tax_code_id, u'lu_tct_m' + parent_code, tax_code_id.replace('lu_tax_code_template_', u''), u'1', u'', pycompat.to_text(name), u'' ])