def export_to_csv(
table,
filename_or_fobj=None,
encoding="utf-8",
dialect=unicodecsv.excel,
batch_size=100,
callback=None,
*args,
**kwargs
):
"""Export a `rows.Table` to a CSV file.
If a file-like object is provided it MUST be in binary mode, like in
`open(filename, mode='wb')`.
If not filename/fobj is provided, the function returns a string with CSV
contents.
"""
# TODO: will work only if table.fields is OrderedDict
# TODO: should use fobj? What about creating a method like json.dumps?
if filename_or_fobj is not None:
_, fobj = get_filename_and_fobj(filename_or_fobj, mode="wb")
else:
fobj = BytesIO()
# TODO: may use `io.BufferedWriter` instead of `ipartition` so user can
# choose the real size (in Bytes) when to flush to the file system, instead
# number of rows
writer = unicodecsv.writer(fobj, encoding=encoding, dialect=dialect)
if callback is None:
for batch in ipartition(serialize(table, *args, **kwargs), batch_size):
writer.writerows(batch)
else:
serialized = serialize(table, *args, **kwargs)
writer.writerow(next(serialized)) # First, write the header
total = 0
for batch in ipartition(serialized, batch_size):
writer.writerows(batch)
total += len(batch)
callback(total)
if filename_or_fobj is not None:
fobj.flush()
return fobj
else:
fobj.seek(0)
result = fobj.read()
fobj.close()
return result
def export_to_csv(table,
filename_or_fobj=None,
encoding="utf-8",
dialect=unicodecsv.excel,
batch_size=100,
callback=None,
*args,
**kwargs):
"""Export a `rows.Table` to a CSV file.
If a file-like object is provided it MUST be in binary mode, like in
`open(filename, mode='wb')`.
If not filename/fobj is provided, the function returns a string with CSV
contents.
"""
# TODO: will work only if table.fields is OrderedDict
# TODO: should use fobj? What about creating a method like json.dumps?
if filename_or_fobj is not None:
_, fobj = get_filename_and_fobj(filename_or_fobj, mode="wb")
else:
fobj = BytesIO()
# TODO: may use `io.BufferedWriter` instead of `ipartition` so user can
# choose the real size (in Bytes) when to flush to the file system, instead
# number of rows
writer = unicodecsv.writer(fobj, encoding=encoding, dialect=dialect)
if callback is None:
for batch in ipartition(serialize(table, *args, **kwargs), batch_size):
writer.writerows(batch)
else:
serialized = serialize(table, *args, **kwargs)
writer.writerow(next(serialized)) # First, write the header
total = 0
for batch in ipartition(serialized, batch_size):
writer.writerows(batch)
total += len(batch)
callback(total)
if filename_or_fobj is not None:
fobj.flush()
return fobj
else:
fobj.seek(0)
result = fobj.read()
fobj.close()
return result
def export_to_sqlite(table_obj, filename_or_connection, table_name='rows',
batch_size=100, *args, **kwargs):
# TODO: should add transaction support?
serialized_table = serialize(table_obj, *args, **kwargs)
connection = _get_connection(filename_or_connection)
table_names = [item[0]
for item in connection.execute(SQL_TABLE_NAMES).fetchall()]
table_name = make_unique_name(name=table_name, existing_names=table_names)
field_names = serialized_table.next()
columns = ['{} {}'.format(field_name,
SQLITE_TYPES[table_obj.fields[field_name]])
for field_name in field_names]
sql = SQL_CREATE_TABLE.format(table_name=table_name,
field_types=', '.join(columns))
connection.execute(sql)
columns = ', '.join(field_names)
placeholders = ', '.join(['?' for field in field_names])
insert_sql = SQL_INSERT.format(table_name=table_name,
field_names=columns,
placeholders=placeholders)
field_types = [table_obj.fields[field_name] for field_name in field_names]
for batch in ipartition(serialized_table, batch_size):
rows_values = [[_convert(field_types[index], value)
for index, value in enumerate(row)]
for row in batch]
connection.executemany(insert_sql, rows_values)
connection.commit()
return connection
def export_to_csv(table, filename_or_fobj=None, encoding='utf-8',
dialect=unicodecsv.excel, *args, **kwargs):
'''Export a `rows.Table` to a CSV file
If a file-like object is provided it MUST be in binary mode, like in
`open(filename, mode='wb')`.
If not filename/fobj is provided, the function returns a string with CSV
contents.
'''
# TODO: will work only if table.fields is OrderedDict
# TODO: should use fobj? What about creating a method like json.dumps?
if filename_or_fobj is not None:
_, fobj = get_filename_and_fobj(filename_or_fobj, mode='wb')
else:
fobj = BytesIO()
writer = unicodecsv.writer(fobj, encoding=encoding, dialect=dialect)
for row in serialize(table, *args, **kwargs):
writer.writerow(row)
if filename_or_fobj is not None:
fobj.flush()
return fobj
else:
fobj.seek(0)
result = fobj.read()
fobj.close()
return result
def export_to_txt(table, filename_or_fobj=None, encoding='utf-8', *args, **kwargs):
# TODO: should be able to change DASH, PLUS and PIPE
# TODO: will work only if table.fields is OrderedDict
# TODO: should use fobj? What about creating a method like json.dumps?
kwargs['encoding'] = encoding
serialized_table = serialize(table, *args, **kwargs)
field_names = serialized_table.next()
table_rows = list(serialized_table)
max_sizes = _max_column_sizes(field_names, table_rows)
dashes = [DASH * (max_sizes[field] + 2) for field in field_names]
header = [field.center(max_sizes[field]) for field in field_names]
header = '{} {} {}'.format(PIPE, ' {} '.format(PIPE).join(header), PIPE)
split_line = PLUS + PLUS.join(dashes) + PLUS
result = [split_line, header, split_line]
for row in table_rows:
values = [value.rjust(max_sizes[field_name])
for field_name, value in zip(field_names, row)]
row_data = ' {} '.format(PIPE).join(values)
result.append('{} {} {}'.format(PIPE, row_data, PIPE))
result.extend([split_line, ''])
data = '\n'.join(result).encode(encoding)
return export_data(filename_or_fobj, data)
def export_to_txt(table,
filename_or_fobj=None,
encoding='utf-8',
*args,
**kwargs):
# TODO: should be able to change DASH, PLUS and PIPE
# TODO: will work only if table.fields is OrderedDict
# TODO: should use fobj? What about creating a method like json.dumps?
kwargs['encoding'] = encoding
serialized_table = serialize(table, *args, **kwargs)
field_names = serialized_table.next()
table_rows = list(serialized_table)
max_sizes = _max_column_sizes(field_names, table_rows)
dashes = [DASH * (max_sizes[field] + 2) for field in field_names]
header = [field.center(max_sizes[field]) for field in field_names]
header = '{} {} {}'.format(PIPE, ' {} '.format(PIPE).join(header), PIPE)
split_line = PLUS + PLUS.join(dashes) + PLUS
result = [split_line, header, split_line]
for row in table_rows:
values = [
value.rjust(max_sizes[field_name])
for field_name, value in zip(field_names, row)
]
row_data = ' {} '.format(PIPE).join(values)
result.append('{} {} {}'.format(PIPE, row_data, PIPE))
result.extend([split_line, ''])
data = '\n'.join(result).encode(encoding)
return export_data(filename_or_fobj, data)
def export_to_csv(table,
filename_or_fobj=None,
encoding='utf-8',
dialect=unicodecsv.excel,
*args,
**kwargs):
'''Export a `rows.Table` to a CSV file
If a file-like object is provided it MUST be in binary mode, like in
`open(filename, mode='wb')`.
If not filename/fobj is provided, the function returns a string with CSV
contents.
'''
# TODO: will work only if table.fields is OrderedDict
# TODO: should use fobj? What about creating a method like json.dumps?
if filename_or_fobj is not None:
_, fobj = get_filename_and_fobj(filename_or_fobj, mode='wb')
else:
fobj = BytesIO()
writer = unicodecsv.writer(fobj, encoding=encoding, dialect=dialect)
for row in serialize(table, *args, **kwargs):
writer.writerow(row)
if filename_or_fobj is not None:
fobj.flush()
return fobj
else:
fobj.seek(0)
result = fobj.read()
fobj.close()
return result
def test_serialize(self):
result = plugins_utils.serialize(utils.table)
field_types = utils.table.fields.values()
self.assertEqual(result.next(), utils.table.fields.keys())
for row, expected_row in zip(result, utils.table._rows):
values = [field_type.serialize(value)
for field_type, value in zip(field_types, expected_row)]
self.assertEqual(values, row)
def test_serialize_should_call_prepare_to_export(self, mocked_prepare_to_export):
table = utils.table
kwargs = {"export_fields": 123, "other_parameter": 3.14}
result = plugins_utils.serialize(table, **kwargs)
self.assertFalse(mocked_prepare_to_export.called)
field_names, table_rows = next(result), list(result)
self.assertTrue(mocked_prepare_to_export.called)
self.assertEqual(mocked_prepare_to_export.call_count, 1)
self.assertEqual(mock.call(table, **kwargs), mocked_prepare_to_export.call_args)
def export_to_html(table, filename_or_fobj=None, encoding="utf-8", *args, **kwargs):
"""Export and return rows.Table data to HTML file."""
serialized_table = serialize(table, *args, **kwargs)
fields = next(serialized_table)
result = ["<table>\n\n", " <thead>\n", " <tr>\n"]
header = [" <th> {} </th>\n".format(field) for field in fields]
result.extend(header)
result.extend([" </tr>\n", " </thead>\n", "\n", " <tbody>\n", "\n"])
for index, row in enumerate(serialized_table, start=1):
css_class = "odd" if index % 2 == 1 else "even"
result.append(' <tr class="{}">\n'.format(css_class))
for value in row:
result.extend([" <td> ", escape(value), " </td>\n"])
result.append(" </tr>\n\n")
result.append(" </tbody>\n\n</table>\n")
html = "".join(result).encode(encoding)
return export_data(filename_or_fobj, html, mode="wb")
def export_to_html(table, filename_or_fobj=None, encoding='utf-8', *args,
**kwargs):
serialized_table = serialize(table, *args, **kwargs)
fields = next(serialized_table)
result = ['<table>\n\n', ' <thead>\n', ' <tr>\n']
header = [' <th> {} </th>\n'.format(field) for field in fields]
result.extend(header)
result.extend([' </tr>\n', ' </thead>\n', '\n', ' <tbody>\n', '\n'])
for index, row in enumerate(serialized_table, start=1):
css_class = 'odd' if index % 2 == 1 else 'even'
result.append(' <tr class="{}">\n'.format(css_class))
for value in row:
result.extend([' <td> ', escape(value), ' </td>\n'])
result.append(' </tr>\n\n')
result.append(' </tbody>\n\n</table>\n')
html = ''.join(result).encode(encoding)
return export_data(filename_or_fobj, html, mode='wb')
def export_to_csv(table, filename_or_fobj=None, encoding='utf-8', *args, **kwargs):
# TODO: will work only if table.fields is OrderedDict
# TODO: should use fobj? What about creating a method like json.dumps?
kwargs['encoding'] = encoding
if filename_or_fobj is not None:
_, fobj = get_filename_and_fobj(filename_or_fobj, mode='w')
else:
fobj = BytesIO()
csv_writer = unicodecsv.writer(fobj, encoding=encoding)
map(csv_writer.writerow, serialize(table, *args, **kwargs))
if filename_or_fobj is not None:
fobj.flush()
return fobj
else:
fobj.seek(0)
result = fobj.read()
fobj.close()
return result
def export_to_html(table,
filename_or_fobj=None,
encoding='utf-8',
*args,
**kwargs):
serialized_table = serialize(table, *args, **kwargs)
fields = next(serialized_table)
result = ['<table>\n\n', ' <thead>\n', ' <tr>\n']
header = [' <th> {} </th>\n'.format(field) for field in fields]
result.extend(header)
result.extend([' </tr>\n', ' </thead>\n', '\n', ' <tbody>\n', '\n'])
for index, row in enumerate(serialized_table, start=1):
css_class = 'odd' if index % 2 == 1 else 'even'
result.append(' <tr class="{}">\n'.format(css_class))
for value in row:
result.extend([' <td> ', value, ' </td>\n'])
result.append(' </tr>\n\n')
result.append(' </tbody>\n\n</table>\n')
html = ''.join(result).encode(encoding)
return export_data(filename_or_fobj, html, mode='wb')
def export_to_html(table, filename_or_fobj=None, encoding='utf-8', *args,
**kwargs):
kwargs['encoding'] = encoding
serialized_table = serialize(table, *args, **kwargs)
fields = serialized_table.next()
result = ['<table>\n\n', ' <thead>\n', ' <tr>\n']
header = [' <th> {} </th>\n'.format(field) for field in fields]
result.extend(header)
result.extend([' </tr>\n', ' </thead>\n', '\n', ' <tbody>\n', '\n'])
for index, row in enumerate(serialized_table, start=1):
css_class = 'odd' if index % 2 == 1 else 'even'
result.append(' <tr class="{}">\n'.format(css_class))
for value in row:
result.extend([' <td> ', value, ' </td>\n'])
result.append(' </tr>\n\n')
result.append(' </tbody>\n\n</table>\n')
new_result = [value.encode(encoding) if isinstance(value, unicode)
else value
for value in result]
html = ''.encode(encoding).join(new_result)
return export_data(filename_or_fobj, html)
def export_to_txt(table, filename_or_fobj=None, encoding=None,
*args, **kwargs):
'''Export a `rows.Table` to text
This function can return the result as a string or save into a file (via
filename or file-like object).
`encoding` could be `None` if no filename/file-like object is specified,
then the return type will be `six.text_type`.
'''
# TODO: should be able to change DASH, PLUS and PIPE
# TODO: will work only if table.fields is OrderedDict
serialized_table = serialize(table, *args, **kwargs)
field_names = next(serialized_table)
table_rows = list(serialized_table)
max_sizes = _max_column_sizes(field_names, table_rows)
dashes = [DASH * (max_sizes[field] + 2) for field in field_names]
header = [field.center(max_sizes[field]) for field in field_names]
header = '{} {} {}'.format(PIPE, ' {} '.format(PIPE).join(header), PIPE)
split_line = PLUS + PLUS.join(dashes) + PLUS
result = [split_line, header, split_line]
for row in table_rows:
values = [value.rjust(max_sizes[field_name])
for field_name, value in zip(field_names, row)]
row_data = ' {} '.format(PIPE).join(values)
result.append('{} {} {}'.format(PIPE, row_data, PIPE))
result.extend([split_line, ''])
data = '\n'.join(result)
if encoding is not None:
data = data.encode(encoding)
return export_data(filename_or_fobj, data, mode='wb')
def export_to_txt(table,
filename_or_fobj=None,
encoding=None,
frame_style="ASCII",
safe_none_frame=True,
*args,
**kwargs):
"""Export a `rows.Table` to text.
This function can return the result as a string or save into a file (via
filename or file-like object).
`encoding` could be `None` if no filename/file-like object is specified,
then the return type will be `six.text_type`.
`frame_style`: will select the frame style to be printed around data.
Valid values are: ('None', 'ASCII', 'single', 'double') - ASCII is default.
Warning: no checks are made to check the desired encoding allows the
characters needed by single and double frame styles.
`safe_none_frame`: bool, defaults to True. Affects only output with
frame_style == "None":
column titles are left-aligned and have
whitespace replaced for "_". This enables
the output to be parseable. Otherwise, the generated table will look
prettier but can not be imported back.
"""
# TODO: will work only if table.fields is OrderedDict
frame_style = _parse_frame_style(frame_style)
frame = FRAMES[frame_style.lower()]
serialized_table = serialize(table, *args, **kwargs)
field_names = next(serialized_table)
table_rows = list(serialized_table)
max_sizes = _max_column_sizes(field_names, table_rows)
dashes = [
frame['HORIZONTAL'] * (max_sizes[field] + 2) for field in field_names
]
if frame_style != 'None' or not safe_none_frame:
header = [field.center(max_sizes[field]) for field in field_names]
else:
header = [
field.replace(" ", "_").ljust(max_sizes[field])
for field in field_names
]
header = '{0} {1} {0}'.format(
frame['VERTICAL'], ' {} '.format(frame['VERTICAL']).join(header))
top_split_line = (frame['DOWN AND RIGHT'] +
frame['DOWN AND HORIZONTAL'].join(dashes) +
frame['DOWN AND LEFT'])
body_split_line = (frame['VERTICAL AND RIGHT'] +
frame['VERTICAL AND HORIZONTAL'].join(dashes) +
frame['VERTICAL AND LEFT'])
botton_split_line = (frame['UP AND RIGHT'] +
frame['UP AND HORIZONTAL'].join(dashes) +
frame['UP AND LEFT'])
result = []
if frame_style != 'None':
result += [top_split_line]
result += [header, body_split_line]
for row in table_rows:
values = [
value.rjust(max_sizes[field_name])
for field_name, value in zip(field_names, row)
]
row_data = ' {} '.format(frame['VERTICAL']).join(values)
result.append('{0} {1} {0}'.format(frame['VERTICAL'], row_data))
if frame_style != 'None':
result.append(botton_split_line)
result.append('')
data = '\n'.join(result)
if encoding is not None:
data = data.encode(encoding)
return export_data(filename_or_fobj, data, mode='wb')
def export_to_txt(
table,
filename_or_fobj=None,
encoding=None,
frame_style="ASCII",
safe_none_frame=True,
*args,
**kwargs
):
"""Export a `rows.Table` to text.
This function can return the result as a string or save into a file (via
filename or file-like object).
`encoding` could be `None` if no filename/file-like object is specified,
then the return type will be `six.text_type`.
`frame_style`: will select the frame style to be printed around data.
Valid values are: ('None', 'ASCII', 'single', 'double') - ASCII is default.
Warning: no checks are made to check the desired encoding allows the
characters needed by single and double frame styles.
`safe_none_frame`: bool, defaults to True. Affects only output with
frame_style == "None":
column titles are left-aligned and have
whitespace replaced for "_". This enables
the output to be parseable. Otherwise, the generated table will look
prettier but can not be imported back.
"""
# TODO: will work only if table.fields is OrderedDict
frame_style = _parse_frame_style(frame_style)
frame = FRAMES[frame_style.lower()]
serialized_table = serialize(table, *args, **kwargs)
field_names = next(serialized_table)
table_rows = list(serialized_table)
max_sizes = _max_column_sizes(field_names, table_rows)
dashes = [frame["HORIZONTAL"] * (max_sizes[field] + 2) for field in field_names]
if frame_style != "None" or not safe_none_frame:
header = [field.center(max_sizes[field]) for field in field_names]
else:
header = [
field.replace(" ", "_").ljust(max_sizes[field]) for field in field_names
]
header = "{0} {1} {0}".format(
frame["VERTICAL"], " {} ".format(frame["VERTICAL"]).join(header)
)
top_split_line = (
frame["DOWN AND RIGHT"]
+ frame["DOWN AND HORIZONTAL"].join(dashes)
+ frame["DOWN AND LEFT"]
)
body_split_line = (
frame["VERTICAL AND RIGHT"]
+ frame["VERTICAL AND HORIZONTAL"].join(dashes)
+ frame["VERTICAL AND LEFT"]
)
botton_split_line = (
frame["UP AND RIGHT"]
+ frame["UP AND HORIZONTAL"].join(dashes)
+ frame["UP AND LEFT"]
)
result = []
if frame_style != "None":
result += [top_split_line]
result += [header, body_split_line]
for row in table_rows:
values = [
value.rjust(max_sizes[field_name])
for field_name, value in zip(field_names, row)
]
row_data = " {} ".format(frame["VERTICAL"]).join(values)
result.append("{0} {1} {0}".format(frame["VERTICAL"], row_data))
if frame_style != "None":
result.append(botton_split_line)
result.append("")
data = "\n".join(result)
if encoding is not None:
data = data.encode(encoding)
return export_data(filename_or_fobj, data, mode="wb")
