def test_register_dialect(self):
import _csv
attrs = [('delimiter', ','),
('doublequote', True),
('escapechar', None),
('lineterminator', '\r\n'),
('quotechar', '"'),
('quoting', _csv.QUOTE_MINIMAL),
('skipinitialspace', False),
('strict', False),
]
for changeattr, newvalue in [('delimiter', ':'),
('doublequote', False),
('escapechar', '/'),
('lineterminator', '---\n'),
('quotechar', '%'),
('quoting', _csv.QUOTE_NONNUMERIC),
('skipinitialspace', True),
('strict', True)]:
kwargs = {changeattr: newvalue}
_csv.register_dialect('foo1', **kwargs)
d = _csv.get_dialect('foo1')
assert d.__class__.__name__ == 'Dialect'
for attr, default in attrs:
if attr == changeattr:
expected = newvalue
else:
expected = default
assert getattr(d, attr) == expected
def test_register_dialect(self):
import _csv
attrs = [('delimiter', ','),
('doublequote', True),
('escapechar', None),
('lineterminator', '\r\n'),
('quotechar', '"'),
('quoting', _csv.QUOTE_MINIMAL),
('skipinitialspace', False),
('strict', False),
]
for changeattr, newvalue in [('delimiter', ':'),
('doublequote', False),
('escapechar', '/'),
('lineterminator', '---\n'),
('quotechar', '%'),
('quoting', _csv.QUOTE_NONNUMERIC),
('skipinitialspace', True),
('strict', True)]:
kwargs = {changeattr: newvalue}
_csv.register_dialect('foo1', **kwargs)
d = _csv.get_dialect('foo1')
assert d.__class__.__name__ == 'Dialect'
for attr, default in attrs:
if attr == changeattr:
expected = newvalue
else:
expected = default
assert getattr(d, attr) == expected
def test_register_dialect_base_2(self):
import _csv
class Foo1:
escapechar = '?'
_csv.register_dialect('foo2', Foo1, strict=True)
d2 = _csv.get_dialect('foo2')
assert d2.escapechar == '?'
assert d2.strict == True
def test_register_dialect_base_2(self):
import _csv
class Foo1:
escapechar = '?'
_csv.register_dialect('foo2', Foo1, strict=True)
d2 = _csv.get_dialect('foo2')
assert d2.escapechar == '?'
assert d2.strict == True
def test_register_dialect_base_1(self):
import _csv
_csv.register_dialect('foo1', escapechar='!')
_csv.register_dialect('foo2', 'foo1', strict=True)
d1 = _csv.get_dialect('foo1')
assert d1.escapechar == '!'
assert d1.strict == False
d2 = _csv.get_dialect('foo2')
assert d2.escapechar == '!'
assert d2.strict == True
def test_register_dialect_base_1(self):
import _csv
_csv.register_dialect('foo1', escapechar='!')
_csv.register_dialect('foo2', 'foo1', strict=True)
d1 = _csv.get_dialect('foo1')
assert d1.escapechar == '!'
assert d1.strict == False
d2 = _csv.get_dialect('foo2')
assert d2.escapechar == '!'
assert d2.strict == True
def test_list_dialects(self):
import _csv
lst = _csv.list_dialects()
assert type(lst) is list
assert 'neverseen' not in lst
_csv.register_dialect('neverseen')
lst = _csv.list_dialects()
assert 'neverseen' in lst
_csv.unregister_dialect('neverseen')
lst = _csv.list_dialects()
assert 'neverseen' not in lst
def test_list_dialects(self):
import _csv
lst = _csv.list_dialects()
assert type(lst) is list
assert 'neverseen' not in lst
_csv.register_dialect('neverseen')
lst = _csv.list_dialects()
assert 'neverseen' in lst
_csv.unregister_dialect('neverseen')
lst = _csv.list_dialects()
assert 'neverseen' not in lst
def build_csv(self, browse_object, method, encoding='utf-8'):
register_dialect("logistic_dialect", self._dialect)
csv_file = StringIO()
# https://github.com/jdunck/python-unicodecsv
writer = unicodecsv.writer(csv_file, encoding=encoding,
dialect=self._dialect)
non_compliant_ids = []
res = getattr(self, method)(browse_object, writer, non_compliant_ids)
if res:
csv_file.seek(0)
return (csv_file.read(), non_compliant_ids)
_logger.info("\nStart to read datas to put file "
"for the method '%s'" % method)
else:
return (False, non_compliant_ids)
def test_bool_arg(self):
# boolean arguments take *any* object and use its truth-value
import _csv
_csv.register_dialect('foo1', doublequote=[])
assert _csv.get_dialect('foo1').doublequote == False
_csv.register_dialect('foo1', skipinitialspace=2)
assert _csv.get_dialect('foo1').skipinitialspace == True
_csv.register_dialect('foo1', strict=_csv) # :-/
assert _csv.get_dialect('foo1').strict == True
def test_bool_arg(self):
# boolean arguments take *any* object and use its truth-value
import _csv
_csv.register_dialect('foo1', doublequote=[])
assert _csv.get_dialect('foo1').doublequote == False
_csv.register_dialect('foo1', skipinitialspace=2)
assert _csv.get_dialect('foo1').skipinitialspace == True
_csv.register_dialect('foo1', strict=_csv) # :-/
assert _csv.get_dialect('foo1').strict == True
from _csv import QUOTE_MINIMAL, register_dialect
from collections import defaultdict
import datetime
class sogenactif_dialect(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = '\t'
quotechar = '"'
doublequote = False
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect("sogenactif_dialect", sogenactif_dialect)
def float_or_zero(val):
return float(val.replace(',', '.')) if val else 0.0
def format_date(val):
return datetime.datetime.strptime(val.split(' ')[0], "%d/%m/%Y")
class SogenactifFileParser(FileParser):
"""
Standard parser that use a define format in csv or xls to import into a
bank statement. This is mostely an example of how to proceed to create a new
parser, but will also be useful as it allow to import a basic flat file.
def test_pickle_dialect(self):
import _csv
import copy
_csv.register_dialect('foo')
raises(TypeError, copy.copy, _csv.get_dialect('foo'))
def test_line_terminator(self):
# lineterminator can be the empty string
import _csv
_csv.register_dialect('foo1', lineterminator='')
assert _csv.get_dialect('foo1').lineterminator == ''
def test_unregister_dialect(self):
import _csv
_csv.register_dialect('foo1')
_csv.unregister_dialect('foo1')
raises(_csv.Error, _csv.get_dialect, 'foo1')
raises(_csv.Error, _csv.unregister_dialect, 'foo1')
from csv import Dialect
from _csv import QUOTE_MINIMAL, register_dialect
import codecs
class be2bill_dialect(Dialect):
delimiter = ';'
quotechar = '"'
doublequote = False
skipinitialspace = False
lineterminator = '\n'
quoting = QUOTE_MINIMAL
register_dialect("be2bill_dialect", be2bill_dialect)
class Be2BillFileParser(FileParser):
def __init__(self, parse_name, ftype='csv'):
conversion_dict = {
'ORDERID': unicode,
'AMOUNT': float_or_zero,
'TRANSACTIONID': unicode,
}
super(Be2BillFileParser, self).__init__(
parse_name, ftype=ftype, conversion_dict=conversion_dict,
dialect=be2bill_dialect
)
@classmethod
def writerow_list(self, L):
self.writer.writer.writerow([v.encode(self.encoding) for v in L])
self._unicode_row()
def writerows(self, rows):
for D in rows:
self.writerow(D)
class excel_semicolon(csv.excel):
"""Extends excel Dialect in order to set semicolon as delimiter."""
delimiter = ';'
quoting = QUOTE_ALL
register_dialect("excel_semicolon", excel_semicolon)
class UnicodeWriter:
"""
A CSV writer which will write rows to CSV file "f",
which is encoded in the given encoding.
"""
def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
# Redirect output to a queue
self.queue = cStringIO.StringIO()
self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
self.stream = f
self.encoder = codecs.getincrementalencoder(encoding)()
self.encoding = encoding
return (float(val.replace(',', '.')) if val else 0.0) / 100.
def format_date(val):
return datetime.datetime.strptime(val, "%Y%m%d").strftime('%Y-%m-%d')
class mercanet_dialect(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = '\t'
quotechar = '"'
doublequote = False
skipinitialspace = False
lineterminator = '\n'
quoting = QUOTE_MINIMAL
register_dialect("mercanet_dialect", mercanet_dialect)
class MercanetFileParser(FileParser):
"""Mercanet parser that use a define format in csv or xls to import
account move.
"""
def __init__(self, parse_name, ftype='csv', **kwargs):
conversion_dict = {
"OPERATION_DATE": format_date,
"PAYMENT_DATE": format_date,
"TRANSACTION_ID": ustr,
"OPERATION_NAME": ustr,
"OPERATION_AMOUNT": float_or_zero,
}
def test_line_terminator(self):
# lineterminator can be the empty string
import _csv
_csv.register_dialect('foo1', lineterminator='')
assert _csv.get_dialect('foo1').lineterminator == ''
except TypeError as e:
# We do this for compatibility with py2.3
raise Error(str(e))
class excel(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect("excel", excel)
class excel_tab(excel):
"""Describe the usual properties of Excel-generated TAB-delimited files."""
delimiter = '\t'
register_dialect("excel-tab", excel_tab)
class unix_dialect(Dialect):
"""Describe the usual properties of Unix-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
def _validate(self):
try:
_Dialect(self)
except TypeError as e:
# We do this for compatibility with py2.3
raise Error(str(e))
class excel(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect("excel", excel)
class excel_tab(excel):
"""Describe the usual properties of Excel-generated TAB-delimited files."""
delimiter = '\t'
register_dialect("excel-tab", excel_tab)
class unix_dialect(Dialect):
"""Describe the usual properties of Unix-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\n'
quoting = QUOTE_ALL
register_dialect("unix", unix_dialect)
_Dialect(self)
except TypeError as e:
raise Error(str(e))
class excel(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect('excel', excel)
class excel_tab(excel):
"""Describe the usual properties of Excel-generated TAB-delimited files."""
delimiter = '\t'
register_dialect('excel-tab', excel_tab)
class DictReader:
def __init__(self,
f,
fieldnames=None,
restkey=None,
import csv
import os
from _csv import register_dialect
from stat import S_IREAD, S_IRGRP, S_IROTH, S_IWUSR
import pandas as pd
class excel_backslash(csv.excel):
"""Standard excel dialect that support backslash character."""
escapechar = "\\"
register_dialect("excel-backslash", excel_backslash)
def unlock_file(filename):
"""Ensures the file can be written to."""
if os.path.exists(filename):
os.chmod(filename, S_IWUSR | S_IREAD)
def lock_file(filename):
"""Prevents file from being accidentally edited."""
os.chmod(filename, S_IREAD | S_IRGRP | S_IROTH)
def read_from_csv(csv_files):
for csv_path in csv_files:
# pd.read_csv seems to crash because of entries with backslashes, so this is a workaround
with open(csv_path, 'r', encoding='UTF-8-sig', newline='') as csv_file:
import datetime
from account_statement_base_import.parser.file_parser import FileParser
from csv import Dialect
from _csv import QUOTE_MINIMAL, register_dialect
from collections import defaultdict
import datetime
class sogenactif_dialect(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = '\t'
quotechar = '"'
doublequote = False
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect("sogenactif_dialect", sogenactif_dialect)
def float_or_zero(val):
return float(val.replace(',', '.')) if val else 0.0
def format_date(val):
return datetime.datetime.strptime(val.split(' ')[0], "%d/%m/%Y")
class SogenactifFileParser(FileParser):
"""
Standard parser that use a define format in csv or xls to import into a
bank statement. This is mostely an example of how to proceed to create a new
parser, but will also be useful as it allow to import a basic flat file.
"""
# TODO this code is not fully updated for So Colissimo
# coorection will be firstly defined in v7 branch
class LaposteDialect(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = ';'
quotechar = '"'
escapechar = '\\'
doublequote = False
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect("laposte_dialect", LaposteDialect)
class DepositSlip(orm.Model):
_inherit = "deposit.slip"
def create_header_vals(self, cr, uid, deposit, context=None):
company = deposit.picking_ids[0].company_id
create_date = datetime.strptime(deposit.create_date,
DEFAULT_SERVER_DATETIME_FORMAT)
create_date_format = datetime.strftime(create_date, "%Y%m%d%H%M")
validate_date = datetime.strftime(datetime.now(), "%Y%m%d%H%M")
vals = {
"Type d'enregistrement": "BBB001",
"Identifiant du bordereau": deposit.name,
"Identifiant du client": company.colipostefr_account,
self.stream.write(data)
# empty queue
self.queue.truncate(0)
def writerows(self, rows):
for D in rows:
self.writerow(D)
class excel_semicolon(csv.excel):
"""Extends excel Dialect in order to set semicolon as delimiter."""
delimiter = ';'
quoting = QUOTE_ALL
register_dialect("excel_semicolon", excel_semicolon)
class excel_quote_all(csv.excel):
"""Extends excel Dialect in order to set semicolon as delimiter."""
delimiter = ','
quotechar = '"'
quoting = QUOTE_ALL
register_dialect("excel_quote_all", excel_semicolon)
class UnicodeWriter:
"""
A CSV writer which will write rows to CSV file "f",
from openerp.tools.translate import _
import datetime
from account_statement_base_import.parser.file_parser import FileParser
from csv import Dialect
from _csv import QUOTE_MINIMAL, register_dialect
from collections import defaultdict
class cb_dialect(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = '\t'
quotechar = '"'
doublequote = False
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect("cb_dialect", cb_dialect)
def float4d(val):
return float(val or 0.0)/10000
def float2d(val):
return float(val or 0.0)/100
class CBFileParser(FileParser):
"""
Standard parser that use a define format in csv or xls to import into a
bank statement. This is mostely an example of how to proceed to create a new
parser, but will also be useful as it allow to import a basic flat file.
"""
def test_unregister_dialect(self):
import _csv
_csv.register_dialect('foo1')
_csv.unregister_dialect('foo1')
raises(_csv.Error, _csv.get_dialect, 'foo1')
raises(_csv.Error, _csv.unregister_dialect, 'foo1')
except TypeError as e:
# We do this for compatibility with py2.3
raise Error(str(e))
class Excel(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect("excel", Excel)
class ExcelTab(Excel):
"""Describe the usual properties of Excel-generated TAB-delimited files."""
delimiter = '\t'
register_dialect("excel-tab", ExcelTab)
class UnixDialect(Dialect):
"""Describe the usual properties of Unix-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
# write to the target stream
self.stream.write(data)
# empty queue
self.queue.truncate(0)
def writerows(self, rows):
for D in rows:
self.writerow(D)
class excel_semicolon(csv.excel):
"""Extends excel Dialect in order to set semicolon as delimiter."""
delimiter = ';'
quoting = QUOTE_ALL
register_dialect("excel_semicolon", excel_semicolon)
class excel_quote_all(csv.excel):
"""Extends excel Dialect in order to set semicolon as delimiter."""
delimiter = ','
quotechar = '"'
quoting = QUOTE_ALL
register_dialect("excel_quote_all", excel_semicolon)
class UnicodeWriter:
"""
A CSV writer which will write rows to CSV file "f",
which is encoded in the given encoding.
"""
_Dialect(self)
except TypeError as e:
raise Error(str(e))
class excel(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect('excel', excel)
class excel_tab(excel):
"""Describe the usual properties of Excel-generated TAB-delimited files."""
delimiter = '\t'
register_dialect('excel-tab', excel_tab)
class DictReader:
def __init__(self, f, fieldnames = None, restkey = None, restval = None, dialect = 'excel', *args, **kwds):
self._fieldnames = fieldnames
self.restkey = restkey
self.restval = restval
self.reader = reader(f, dialect, *args, **kwds)
def _validate(self):
try:
_Dialect(self)
except TypeError as e:
raise Error(str(e))
class excel(Dialect):
__qualname__ = 'excel'
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect('excel', excel)
class excel_tab(excel):
__qualname__ = 'excel_tab'
delimiter = '\t'
register_dialect('excel-tab', excel_tab)
class unix_dialect(Dialect):
__qualname__ = 'unix_dialect'
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\n'
quoting = QUOTE_ALL
except TypeError as e:
# We do this for compatibility with py2.3
raise Error(str(e))
class excel(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = str(',')
quotechar = str('"')
doublequote = True
skipinitialspace = False
lineterminator = str('\r\n')
quoting = QUOTE_MINIMAL
register_dialect("excel", excel)
class excel_tab(excel):
"""Describe the usual properties of Excel-generated TAB-delimited files."""
delimiter = str('\t')
register_dialect("excel-tab", excel_tab)
# Guard Sniffer's type checking against builds that exclude complex()
try:
complex
except NameError:
complex = float
# write to the target stream
self.stream.write(data)
# empty queue
self.queue.truncate(0)
def writerows(self, rows):
for D in rows:
self.writerow(D)
class excel_semicolon(csv.excel):
"""Extends excel Dialect in order to set semicolon as delimiter."""
delimiter = ';'
quoting = QUOTE_ALL
register_dialect("excel_semicolon", excel_semicolon)
class UnicodeWriter:
"""
A CSV writer which will write rows to CSV file "f",
which is encoded in the given encoding.
"""
def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
# Redirect output to a queue
self.queue = cStringIO.StringIO()
self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
self.stream = f
self.encoder = codecs.getincrementalencoder(encoding)()
def _validate(self):
try:
_Dialect(self)
except TypeError, e:
# We do this for compatibility with py2.3
raise Error(str(e))
class excel(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = ','
quotechar = '"'
doublequote = True
skipinitialspace = False
lineterminator = '\r\n'
quoting = QUOTE_MINIMAL
register_dialect("excel", excel)
class excel_tab(excel):
"""Describe the usual properties of Excel-generated TAB-delimited files."""
delimiter = '\t'
register_dialect("excel-tab", excel_tab)
class DictReader:
def __init__(self, f, fieldnames=None, restkey=None, restval=None,
dialect="excel", *args, **kwds):
self.fieldnames = fieldnames # list of keys for the dict
self.restkey = restkey # key to catch long rows
self.restval = restval # default value for short rows
self.reader = reader(f, dialect, *args, **kwds)
def format_date(val):
return datetime.datetime.strptime(val, "%Y%m%d")
class AtosDialect(Dialect):
"""Describe the usual properties of Excel-generated CSV files."""
delimiter = '\t'
quotechar = '"'
doublequote = False
skipinitialspace = False
lineterminator = '\n'
quoting = QUOTE_MINIMAL
register_dialect("atos_dialect", AtosDialect)
class AtosFileParser(FileParser):
def __init__(self, journal, ftype='csv', **kwargs):
extra_fields = {
"OPERATION_DATE": format_date,
"PAYMENT_DATE": unicode,
"TRANSACTION_ID": unicode,
"OPERATION_NAME": unicode,
"OPERATION_AMOUNT": float_or_zero,
}
self.refund_amount = None
super(AtosFileParser, self).__init__(journal,
ftype=ftype,
extra_fields=extra_fields,
