def test_register_dialect(self): import _csv attrs = [('delimiter', ','), ('doublequote', True), ('escapechar', None), ('lineterminator', '\r\n'), ('quotechar', '"'), ('quoting', _csv.QUOTE_MINIMAL), ('skipinitialspace', False), ('strict', False), ] for changeattr, newvalue in [('delimiter', ':'), ('doublequote', False), ('escapechar', '/'), ('lineterminator', '---\n'), ('quotechar', '%'), ('quoting', _csv.QUOTE_NONNUMERIC), ('skipinitialspace', True), ('strict', True)]: kwargs = {changeattr: newvalue} _csv.register_dialect('foo1', **kwargs) d = _csv.get_dialect('foo1') assert d.__class__.__name__ == 'Dialect' for attr, default in attrs: if attr == changeattr: expected = newvalue else: expected = default assert getattr(d, attr) == expected
def test_register_dialect_base_2(self): import _csv class Foo1: escapechar = '?' _csv.register_dialect('foo2', Foo1, strict=True) d2 = _csv.get_dialect('foo2') assert d2.escapechar == '?' assert d2.strict == True
def test_register_dialect_base_1(self): import _csv _csv.register_dialect('foo1', escapechar='!') _csv.register_dialect('foo2', 'foo1', strict=True) d1 = _csv.get_dialect('foo1') assert d1.escapechar == '!' assert d1.strict == False d2 = _csv.get_dialect('foo2') assert d2.escapechar == '!' assert d2.strict == True
def test_list_dialects(self): import _csv lst = _csv.list_dialects() assert type(lst) is list assert 'neverseen' not in lst _csv.register_dialect('neverseen') lst = _csv.list_dialects() assert 'neverseen' in lst _csv.unregister_dialect('neverseen') lst = _csv.list_dialects() assert 'neverseen' not in lst
def build_csv(self, browse_object, method, encoding='utf-8'): register_dialect("logistic_dialect", self._dialect) csv_file = StringIO() # https://github.com/jdunck/python-unicodecsv writer = unicodecsv.writer(csv_file, encoding=encoding, dialect=self._dialect) non_compliant_ids = [] res = getattr(self, method)(browse_object, writer, non_compliant_ids) if res: csv_file.seek(0) return (csv_file.read(), non_compliant_ids) _logger.info("\nStart to read datas to put file " "for the method '%s'" % method) else: return (False, non_compliant_ids)
def test_bool_arg(self): # boolean arguments take *any* object and use its truth-value import _csv _csv.register_dialect('foo1', doublequote=[]) assert _csv.get_dialect('foo1').doublequote == False _csv.register_dialect('foo1', skipinitialspace=2) assert _csv.get_dialect('foo1').skipinitialspace == True _csv.register_dialect('foo1', strict=_csv) # :-/ assert _csv.get_dialect('foo1').strict == True
from _csv import QUOTE_MINIMAL, register_dialect from collections import defaultdict import datetime class sogenactif_dialect(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = '\t' quotechar = '"' doublequote = False skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("sogenactif_dialect", sogenactif_dialect) def float_or_zero(val): return float(val.replace(',', '.')) if val else 0.0 def format_date(val): return datetime.datetime.strptime(val.split(' ')[0], "%d/%m/%Y") class SogenactifFileParser(FileParser): """ Standard parser that use a define format in csv or xls to import into a bank statement. This is mostely an example of how to proceed to create a new parser, but will also be useful as it allow to import a basic flat file.
def test_pickle_dialect(self): import _csv import copy _csv.register_dialect('foo') raises(TypeError, copy.copy, _csv.get_dialect('foo'))
def test_line_terminator(self): # lineterminator can be the empty string import _csv _csv.register_dialect('foo1', lineterminator='') assert _csv.get_dialect('foo1').lineterminator == ''
def test_unregister_dialect(self): import _csv _csv.register_dialect('foo1') _csv.unregister_dialect('foo1') raises(_csv.Error, _csv.get_dialect, 'foo1') raises(_csv.Error, _csv.unregister_dialect, 'foo1')
from csv import Dialect from _csv import QUOTE_MINIMAL, register_dialect import codecs class be2bill_dialect(Dialect): delimiter = ';' quotechar = '"' doublequote = False skipinitialspace = False lineterminator = '\n' quoting = QUOTE_MINIMAL register_dialect("be2bill_dialect", be2bill_dialect) class Be2BillFileParser(FileParser): def __init__(self, parse_name, ftype='csv'): conversion_dict = { 'ORDERID': unicode, 'AMOUNT': float_or_zero, 'TRANSACTIONID': unicode, } super(Be2BillFileParser, self).__init__( parse_name, ftype=ftype, conversion_dict=conversion_dict, dialect=be2bill_dialect ) @classmethod
def writerow_list(self, L): self.writer.writer.writerow([v.encode(self.encoding) for v in L]) self._unicode_row() def writerows(self, rows): for D in rows: self.writerow(D) class excel_semicolon(csv.excel): """Extends excel Dialect in order to set semicolon as delimiter.""" delimiter = ';' quoting = QUOTE_ALL register_dialect("excel_semicolon", excel_semicolon) class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. """ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): # Redirect output to a queue self.queue = cStringIO.StringIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)() self.encoding = encoding
return (float(val.replace(',', '.')) if val else 0.0) / 100. def format_date(val): return datetime.datetime.strptime(val, "%Y%m%d").strftime('%Y-%m-%d') class mercanet_dialect(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = '\t' quotechar = '"' doublequote = False skipinitialspace = False lineterminator = '\n' quoting = QUOTE_MINIMAL register_dialect("mercanet_dialect", mercanet_dialect) class MercanetFileParser(FileParser): """Mercanet parser that use a define format in csv or xls to import account move. """ def __init__(self, parse_name, ftype='csv', **kwargs): conversion_dict = { "OPERATION_DATE": format_date, "PAYMENT_DATE": format_date, "TRANSACTION_ID": ustr, "OPERATION_NAME": ustr, "OPERATION_AMOUNT": float_or_zero, }
except TypeError as e: # We do this for compatibility with py2.3 raise Error(str(e)) class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("excel", excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' register_dialect("excel-tab", excel_tab) class unix_dialect(Dialect): """Describe the usual properties of Unix-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True
def _validate(self): try: _Dialect(self) except TypeError as e: # We do this for compatibility with py2.3 raise Error(str(e)) class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("excel", excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' register_dialect("excel-tab", excel_tab) class unix_dialect(Dialect): """Describe the usual properties of Unix-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\n' quoting = QUOTE_ALL register_dialect("unix", unix_dialect)
_Dialect(self) except TypeError as e: raise Error(str(e)) class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect('excel', excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' register_dialect('excel-tab', excel_tab) class DictReader: def __init__(self, f, fieldnames=None, restkey=None,
import csv import os from _csv import register_dialect from stat import S_IREAD, S_IRGRP, S_IROTH, S_IWUSR import pandas as pd class excel_backslash(csv.excel): """Standard excel dialect that support backslash character.""" escapechar = "\\" register_dialect("excel-backslash", excel_backslash) def unlock_file(filename): """Ensures the file can be written to.""" if os.path.exists(filename): os.chmod(filename, S_IWUSR | S_IREAD) def lock_file(filename): """Prevents file from being accidentally edited.""" os.chmod(filename, S_IREAD | S_IRGRP | S_IROTH) def read_from_csv(csv_files): for csv_path in csv_files: # pd.read_csv seems to crash because of entries with backslashes, so this is a workaround with open(csv_path, 'r', encoding='UTF-8-sig', newline='') as csv_file:
import datetime from account_statement_base_import.parser.file_parser import FileParser from csv import Dialect from _csv import QUOTE_MINIMAL, register_dialect from collections import defaultdict import datetime class sogenactif_dialect(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = '\t' quotechar = '"' doublequote = False skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("sogenactif_dialect", sogenactif_dialect) def float_or_zero(val): return float(val.replace(',', '.')) if val else 0.0 def format_date(val): return datetime.datetime.strptime(val.split(' ')[0], "%d/%m/%Y") class SogenactifFileParser(FileParser): """ Standard parser that use a define format in csv or xls to import into a bank statement. This is mostely an example of how to proceed to create a new parser, but will also be useful as it allow to import a basic flat file. """
# TODO this code is not fully updated for So Colissimo # coorection will be firstly defined in v7 branch class LaposteDialect(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ';' quotechar = '"' escapechar = '\\' doublequote = False skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("laposte_dialect", LaposteDialect) class DepositSlip(orm.Model): _inherit = "deposit.slip" def create_header_vals(self, cr, uid, deposit, context=None): company = deposit.picking_ids[0].company_id create_date = datetime.strptime(deposit.create_date, DEFAULT_SERVER_DATETIME_FORMAT) create_date_format = datetime.strftime(create_date, "%Y%m%d%H%M") validate_date = datetime.strftime(datetime.now(), "%Y%m%d%H%M") vals = { "Type d'enregistrement": "BBB001", "Identifiant du bordereau": deposit.name, "Identifiant du client": company.colipostefr_account,
self.stream.write(data) # empty queue self.queue.truncate(0) def writerows(self, rows): for D in rows: self.writerow(D) class excel_semicolon(csv.excel): """Extends excel Dialect in order to set semicolon as delimiter.""" delimiter = ';' quoting = QUOTE_ALL register_dialect("excel_semicolon", excel_semicolon) class excel_quote_all(csv.excel): """Extends excel Dialect in order to set semicolon as delimiter.""" delimiter = ',' quotechar = '"' quoting = QUOTE_ALL register_dialect("excel_quote_all", excel_semicolon) class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f",
from openerp.tools.translate import _ import datetime from account_statement_base_import.parser.file_parser import FileParser from csv import Dialect from _csv import QUOTE_MINIMAL, register_dialect from collections import defaultdict class cb_dialect(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = '\t' quotechar = '"' doublequote = False skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("cb_dialect", cb_dialect) def float4d(val): return float(val or 0.0)/10000 def float2d(val): return float(val or 0.0)/100 class CBFileParser(FileParser): """ Standard parser that use a define format in csv or xls to import into a bank statement. This is mostely an example of how to proceed to create a new parser, but will also be useful as it allow to import a basic flat file. """
except TypeError as e: # We do this for compatibility with py2.3 raise Error(str(e)) class Excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("excel", Excel) class ExcelTab(Excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' register_dialect("excel-tab", ExcelTab) class UnixDialect(Dialect): """Describe the usual properties of Unix-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True
# write to the target stream self.stream.write(data) # empty queue self.queue.truncate(0) def writerows(self, rows): for D in rows: self.writerow(D) class excel_semicolon(csv.excel): """Extends excel Dialect in order to set semicolon as delimiter.""" delimiter = ';' quoting = QUOTE_ALL register_dialect("excel_semicolon", excel_semicolon) class excel_quote_all(csv.excel): """Extends excel Dialect in order to set semicolon as delimiter.""" delimiter = ',' quotechar = '"' quoting = QUOTE_ALL register_dialect("excel_quote_all", excel_semicolon) class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. """
_Dialect(self) except TypeError as e: raise Error(str(e)) class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect('excel', excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' register_dialect('excel-tab', excel_tab) class DictReader: def __init__(self, f, fieldnames = None, restkey = None, restval = None, dialect = 'excel', *args, **kwds): self._fieldnames = fieldnames self.restkey = restkey self.restval = restval self.reader = reader(f, dialect, *args, **kwds)
def _validate(self): try: _Dialect(self) except TypeError as e: raise Error(str(e)) class excel(Dialect): __qualname__ = 'excel' delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect('excel', excel) class excel_tab(excel): __qualname__ = 'excel_tab' delimiter = '\t' register_dialect('excel-tab', excel_tab) class unix_dialect(Dialect): __qualname__ = 'unix_dialect' delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\n' quoting = QUOTE_ALL
except TypeError as e: # We do this for compatibility with py2.3 raise Error(str(e)) class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = str(',') quotechar = str('"') doublequote = True skipinitialspace = False lineterminator = str('\r\n') quoting = QUOTE_MINIMAL register_dialect("excel", excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = str('\t') register_dialect("excel-tab", excel_tab) # Guard Sniffer's type checking against builds that exclude complex() try: complex except NameError: complex = float
# write to the target stream self.stream.write(data) # empty queue self.queue.truncate(0) def writerows(self, rows): for D in rows: self.writerow(D) class excel_semicolon(csv.excel): """Extends excel Dialect in order to set semicolon as delimiter.""" delimiter = ';' quoting = QUOTE_ALL register_dialect("excel_semicolon", excel_semicolon) class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. """ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): # Redirect output to a queue self.queue = cStringIO.StringIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)()
def _validate(self): try: _Dialect(self) except TypeError, e: # We do this for compatibility with py2.3 raise Error(str(e)) class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = ',' quotechar = '"' doublequote = True skipinitialspace = False lineterminator = '\r\n' quoting = QUOTE_MINIMAL register_dialect("excel", excel) class excel_tab(excel): """Describe the usual properties of Excel-generated TAB-delimited files.""" delimiter = '\t' register_dialect("excel-tab", excel_tab) class DictReader: def __init__(self, f, fieldnames=None, restkey=None, restval=None, dialect="excel", *args, **kwds): self.fieldnames = fieldnames # list of keys for the dict self.restkey = restkey # key to catch long rows self.restval = restval # default value for short rows self.reader = reader(f, dialect, *args, **kwds)
def format_date(val): return datetime.datetime.strptime(val, "%Y%m%d") class AtosDialect(Dialect): """Describe the usual properties of Excel-generated CSV files.""" delimiter = '\t' quotechar = '"' doublequote = False skipinitialspace = False lineterminator = '\n' quoting = QUOTE_MINIMAL register_dialect("atos_dialect", AtosDialect) class AtosFileParser(FileParser): def __init__(self, journal, ftype='csv', **kwargs): extra_fields = { "OPERATION_DATE": format_date, "PAYMENT_DATE": unicode, "TRANSACTION_ID": unicode, "OPERATION_NAME": unicode, "OPERATION_AMOUNT": float_or_zero, } self.refund_amount = None super(AtosFileParser, self).__init__(journal, ftype=ftype, extra_fields=extra_fields,