def test_register_kwargs(self):
name = "fedcba"
csv.register_dialect(name, delimiter=";")
try:
self.assertEqual(csv.get_dialect(name).delimiter, ";")
self.assertEqual([["X", "Y", "Z"]], list(csv.reader(["X;Y;Z"], name)))
finally:
csv.unregister_dialect(name)
def test_register_kwargs(self):
name = 'fedcba'
csv.register_dialect(name, delimiter=';')
try:
self.assertEqual(csv.get_dialect(name).delimiter, ';')
self.assertEqual([['X', 'Y', 'Z']], list(csv.reader(['X;Y;Z'], name)))
finally:
csv.unregister_dialect(name)
def test_register_kwargs(self):
name = 'fedcba'
csv.register_dialect(name, delimiter=';')
try:
self.assertEqual(csv.get_dialect(name).delimiter, ';')
self.assertEqual([['X', 'Y', 'Z']],
list(csv.reader(['X;Y;Z'], name)))
finally:
csv.unregister_dialect(name)
def test_registry(self):
class myexceltsv(csv.excel):
delimiter = "\t"
name = "myexceltsv"
expected_dialects = csv.list_dialects() + [name]
expected_dialects.sort()
csv.register_dialect(name, myexceltsv)
self.addCleanup(csv.unregister_dialect, name)
self.assertEqual(csv.get_dialect(name).delimiter, '\t')
got_dialects = sorted(csv.list_dialects())
self.assertEqual(expected_dialects, got_dialects)
def test_registry(self):
class myexceltsv(csv.excel):
delimiter = "\t"
name = "myexceltsv"
expected_dialects = csv.list_dialects() + [name]
expected_dialects.sort()
csv.register_dialect(name, myexceltsv)
try:
self.assertEqual(csv.get_dialect(name).delimiter, '\t')
got_dialects = sorted(csv.list_dialects())
self.assertEqual(expected_dialects, got_dialects)
finally:
csv.unregister_dialect(name)
def write_to_csv(self):
csv.register_dialect(name=u"export",
delimiter=self.opts.delimiter.decode('utf-8'),
quotechar=self.opts.quotechar.decode('utf-8'),
doublequote=True,
skipinitialspace=False,
lineterminator=u'\r\n',
quoting=csv.QUOTE_ALL)
if self.num_results > 0:
self.num_results = sum(1 for line in codecs.open(
self.tmp_file, mode='r', encoding='utf-8'))
if self.num_results > 0:
output_file = codecs.open(self.opts.output_file,
mode='a',
encoding='utf-8')
csv_writer = csv.DictWriter(output_file,
fieldnames=self.csv_headers,
dialect="export")
csv_writer.writeheader()
timer = 0
widgets = [
'Write to csv ',
progressbar.Bar(left='[', marker='#', right=']'),
progressbar.FormatLabel(' [%(value)i/%(max)i] ['),
progressbar.Percentage(),
progressbar.FormatLabel('] [%(elapsed)s] ['),
progressbar.ETA(), '] [',
progressbar.FileTransferSpeed(unit='lines'), ']'
]
bar = progressbar.ProgressBar(widgets=widgets,
maxval=self.num_results).start()
for line in codecs.open(self.tmp_file,
mode='r',
encoding='utf-8'):
timer += 1
bar.update(timer)
csv_writer.writerow(json.loads(line))
output_file.close()
bar.finish()
else:
print('There is no docs with selected field(s): {}.'.format(
','.join(self.opts.fields)))
os.remove(self.tmp_file)
def test_dialect_apply(self):
class testA(csv.excel):
delimiter = "\t"
class testB(csv.excel):
delimiter = ":"
class testC(csv.excel):
delimiter = "|"
class testUni(csv.excel):
delimiter = "\u039B"
csv.register_dialect('testC', testC)
try:
self.compare_dialect_123("1,2,3\r\n")
self.compare_dialect_123("1\t2\t3\r\n", testA)
self.compare_dialect_123("1:2:3\r\n", dialect=testB())
self.compare_dialect_123("1|2|3\r\n", dialect='testC')
self.compare_dialect_123("1;2;3\r\n", dialect=testA,
delimiter=';')
self.compare_dialect_123("1\u039B2\u039B3\r\n",
dialect=testUni)
finally:
csv.unregister_dialect('testC')
def test_dialect_apply(self):
class testA(csv.excel):
delimiter = "\t"
class testB(csv.excel):
delimiter = ":"
class testC(csv.excel):
delimiter = "|"
class testUni(csv.excel):
delimiter = "\u039B"
csv.register_dialect('testC', testC)
try:
self.compare_dialect_123("1,2,3\r\n")
self.compare_dialect_123("1\t2\t3\r\n", testA)
self.compare_dialect_123("1:2:3\r\n", dialect=testB())
self.compare_dialect_123("1|2|3\r\n", dialect='testC')
self.compare_dialect_123("1;2;3\r\n", dialect=testA, delimiter=';')
self.compare_dialect_123("1\u039B2\u039B3\r\n", dialect=testUni)
finally:
csv.unregister_dialect('testC')
OMEGAT_FIELDNAMES = ["source", "target", "comment"]
"""Field names for an OmegaT glossary unit"""
class OmegaTDialect(csv.Dialect):
"""Describe the properties of an OmegaT generated TAB-delimited glossary
file.
"""
delimiter = "\t"
lineterminator = "\r\n"
quoting = csv.QUOTE_NONE
csv.register_dialect("omegat", OmegaTDialect)
@six.python_2_unicode_compatible
class OmegaTUnit(base.TranslationUnit):
"""An OmegaT glossary unit"""
def __init__(self, source=None):
self._dict = {}
if source:
self.source = source
super(OmegaTUnit, self).__init__(source)
def getdict(self):
"""Get the dictionary of values for a OmegaT line"""
return self._dict
def _unescape(string):
if string:
string = string.replace("\\n", "\n").replace("\\t", "\t").replace(r"\n", r"\\n")
return string
class CatkeysDialect(csv.Dialect):
"""Describe the properties of a catkeys generated TAB-delimited file."""
delimiter = "\t"
lineterminator = "\n"
quoting = csv.QUOTE_NONE
csv.register_dialect("catkeys", CatkeysDialect)
class CatkeysHeader(object):
"""A catkeys translation memory header"""
def __init__(self, header=None):
self._header_dict = {}
if not header:
self._header_dict = self._create_default_header()
elif isinstance(header, dict):
self._header_dict = header
def _create_default_header(self):
"""Create a default catkeys header"""
defaultheader = FIELDNAMES_HEADER_DEFAULTS.copy()
import six
if six.PY2:
from backports import csv
else:
import csv
from translate.storage import base
class DefaultDialect(csv.excel):
skipinitialspace = True
quoting = csv.QUOTE_ALL
escapechar = '\\'
csv.register_dialect('default', DefaultDialect)
@six.python_2_unicode_compatible
class csvunit(base.TranslationUnit):
spreadsheetescapes = [("+", "\\+"), ("-", "\\-"), ("=", "\\="), ("'", "\\'")]
def __init__(self, source=None):
super(csvunit, self).__init__(source)
self.location = ""
self.source = source or ""
self.target = ""
self.id = ""
self.fuzzy = 'False'
self.developer_comments = ""
self.translator_comments = ""
class UtxDialect(csv.Dialect):
"""Describe the properties of an UTX generated TAB-delimited dictionary
file.
"""
delimiter = "\t"
# The spec says \r\n but there are older version < 1.0 with just \n
# FIXME if we find older specs then lets see if we can support these
# differences
lineterminator = "\r\n"
quoting = csv.QUOTE_NONE
csv.register_dialect("utx", UtxDialect)
class UtxHeader:
"""A UTX header entry
A UTX header is a single line that looks like this::
#UTX-S <version>; < source language >/< target language>;
<date created>; <optional fields (creator, license, etc.)>
Where::
- UTX-S version is currently 1.00.
- Source language/target language: ISO 639, 3166 formats.
In the case of monolingual dictionary, target language should be
omitted.
- Date created: ISO 8601 format
import six
if six.PY2:
from backports import csv
else:
import csv
from translate.storage import base
class DefaultDialect(csv.excel):
skipinitialspace = True
quoting = csv.QUOTE_NONNUMERIC
escapechar = '\\'
csv.register_dialect('default', DefaultDialect)
@six.python_2_unicode_compatible
class csvunit(base.TranslationUnit):
spreadsheetescapes = [("+", "\\+"), ("-", "\\-"), ("=", "\\="),
("'", "\\'")]
def __init__(self, source=None):
super(csvunit, self).__init__(source)
self.location = ""
self.source = source or ""
self.target = ""
self.id = ""
self.fuzzy = 'False'
self.developer_comments = ""
OMEGAT_FIELDNAMES = ["source", "target", "comment"]
"""Field names for an OmegaT glossary unit"""
class OmegaTDialect(csv.Dialect):
"""Describe the properties of an OmegaT generated TAB-delimited glossary
file.
"""
delimiter = "\t"
lineterminator = "\r\n"
quoting = csv.QUOTE_NONE
csv.register_dialect("omegat", OmegaTDialect)
@six.python_2_unicode_compatible
class OmegaTUnit(base.TranslationUnit):
"""An OmegaT glossary unit"""
def __init__(self, source=None):
self._dict = {}
if source:
self.source = source
super(OmegaTUnit, self).__init__(source)
def getdict(self):
"""Get the dictionary of values for a OmegaT line"""
return self._dict
class UtxDialect(csv.Dialect):
"""Describe the properties of an UTX generated TAB-delimited dictionary
file.
"""
delimiter = "\t"
# The spec says \r\n but there are older version < 1.0 with just \n
# FIXME if we find older specs then lets see if we can support these
# differences
lineterminator = "\r\n"
quoting = csv.QUOTE_NONE
csv.register_dialect("utx", UtxDialect)
class UtxHeader:
"""A UTX header entry
A UTX header is a single line that looks like this::
#UTX-S <version>; < source language >/< target language>;
<date created>; <optional fields (creator, license, etc.)>
Where::
- UTX-S version is currently 1.00.
- Source language/target language: ISO 639, 3166 formats.
In the case of monolingual dictionary, target language should be
omitted.
- Date created: ISO 8601 format
if string:
for code, char in WF_ESCAPE_MAP:
string = string.replace(code, char)
string = string.replace("\\n", "\n").replace("\\t", "\t")
return string
class WordfastDialect(csv.Dialect):
"""Describe the properties of a Wordfast generated TAB-delimited file."""
delimiter = "\t"
lineterminator = "\r\n"
quoting = csv.QUOTE_NONE
csv.register_dialect("wordfast", WordfastDialect)
class WordfastTime(object):
"""Manages time stamps in the Wordfast format of YYYYMMDD~hhmmss"""
def __init__(self, newtime=None):
self._time = None
if not newtime:
self.time = None
elif isinstance(newtime, six.string_types):
self.timestring = newtime
elif isinstance(newtime, time.struct_time):
self.time = newtime
def get_timestring(self):
if string:
string = string.replace("\\n",
"\n").replace("\\t",
"\t").replace(r"\n", r"\\n")
return string
class CatkeysDialect(csv.Dialect):
"""Describe the properties of a catkeys generated TAB-delimited file."""
delimiter = "\t"
lineterminator = "\n"
quoting = csv.QUOTE_NONE
csv.register_dialect("catkeys", CatkeysDialect)
class CatkeysHeader(object):
"""A catkeys translation memory header"""
def __init__(self, header=None):
self._header_dict = {}
if not header:
self._header_dict = self._create_default_header()
elif isinstance(header, dict):
self._header_dict = header
def _create_default_header(self):
"""Create a default catkeys header"""
defaultheader = FIELDNAMES_HEADER_DEFAULTS.copy()
return defaultheader
