def test_export_to_txt_fobj(self):
# TODO: may test with codecs.open passing an encoding
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
rows.export_to_txt(utils.table, temp.file)
self.assert_file_contents_equal(temp.name, self.filename)
def test_export_to_txt_fobj(self):
# TODO: may test with codecs.open passing an encoding
# TODO: may test file contents
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
rows.export_to_txt(utils.table, temp.file, encoding="utf-8")
table = rows.import_from_txt(temp.name, encoding="utf-8")
self.assert_table_equal(table, utils.table)
def test_export_to_txt_fobj(self):
# TODO: may test with codecs.open passing an encoding
# TODO: may test file contents
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
rows.export_to_txt(utils.table, temp.file, encoding='utf-8')
table = rows.import_from_txt(temp.name, encoding='utf-8')
self.assert_table_equal(table, utils.table)
def test_issue_168(self):
temp = tempfile.NamedTemporaryFile(delete=False)
filename = "{}.{}".format(temp.name, self.file_extension)
self.files_to_delete.append(filename)
table = rows.Table(fields=OrderedDict([("jsoncolumn", rows.fields.JSONField)]))
table.append({"jsoncolumn": '{"python": 42}'})
rows.export_to_txt(table, filename, encoding="utf-8")
table2 = rows.import_from_txt(filename, encoding="utf-8")
self.assert_table_equal(table, table2)
def test_export_to_txt_filename(self):
# TODO: may test file contents
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
rows.export_to_txt(utils.table, temp.name, encoding='utf-8')
table = rows.import_from_txt(temp.name, encoding='utf-8')
self.assert_table_equal(table, utils.table)
with open(temp.name, mode='rb') as fobj:
content = fobj.read()
self.assertEqual(content[-10:].count(b'\n'), 1)
def test_issue_168(self):
temp = tempfile.NamedTemporaryFile(delete=False)
filename = '{}.{}'.format(temp.name, self.file_extension)
self.files_to_delete.append(filename)
table = rows.Table(fields=OrderedDict([('jsoncolumn',
rows.fields.JSONField)]))
table.append({'jsoncolumn': '{"python": 42}'})
rows.export_to_txt(table, filename, encoding='utf-8')
table2 = rows.import_from_txt(filename, encoding='utf-8')
self.assert_table_equal(table, table2)
def test_export_to_txt_filename(self):
# TODO: may test file contents
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
rows.export_to_txt(utils.table, temp.name, encoding="utf-8")
table = rows.import_from_txt(temp.name, encoding="utf-8")
self.assert_table_equal(table, utils.table)
with open(temp.name, mode="rb") as fobj:
content = fobj.read()
self.assertEqual(content[-10:].count(b"\n"), 1)
def query(input_encoding, output_encoding, input_locale, output_locale,
verify_ssl, fields, output, query, sources):
# TODO: may move all 'destination' to '--output'
# TODO: may use sys.stdout.encoding if output_file = '-'
output_encoding = output_encoding or sys.stdout.encoding or \
DEFAULT_OUTPUT_ENCODING
if not query.lower().startswith('select'):
field_names = '*' if fields is None else fields
table_names = ', '.join(
['table{}'.format(index) for index in range(1,
len(sources) + 1)])
query = 'SELECT {} FROM {} WHERE {}'.format(field_names, table_names,
query)
if input_locale is not None:
with rows.locale_context(input_locale):
tables = [
_import_table(source,
encoding=input_encoding,
verify_ssl=verify_ssl) for source in sources
]
else:
tables = [
_import_table(source,
encoding=input_encoding,
verify_ssl=verify_ssl) for source in sources
]
sqlite_connection = rows.export_to_sqlite(tables[0],
':memory:',
table_name='table1')
for index, table in enumerate(tables[1:], start=2):
rows.export_to_sqlite(table,
sqlite_connection,
table_name='table{}'.format(index))
result = rows.import_from_sqlite(sqlite_connection, query=query)
if output is None:
fobj = BytesIO()
if output_locale is not None:
with rows.locale_context(output_locale):
rows.export_to_txt(result, fobj, encoding=output_encoding)
else:
rows.export_to_txt(result, fobj, encoding=output_encoding)
fobj.seek(0)
click.echo(fobj.read())
else:
if output_locale is not None:
with rows.locale_context(output_locale):
export_to_uri(output, result, encoding=output_encoding)
else:
export_to_uri(output, result, encoding=output_encoding)
def test_export_to_txt_uses_serialize(self, mocked_serialize):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
kwargs = {"test": 123, "parameter": 3.14}
mocked_serialize.return_value = iter([utils.table.fields.keys()])
rows.export_to_txt(utils.table, temp.name, encoding=self.encoding, **kwargs)
self.assertTrue(mocked_serialize.called)
self.assertEqual(mocked_serialize.call_count, 1)
call = mocked_serialize.call_args
self.assertEqual(call[0], (utils.table,))
self.assertEqual(call[1], kwargs)
def test_export_to_txt_uses_serialize(self, mocked_serialize):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
kwargs = {'test': 123, 'parameter': 3.14, }
mocked_serialize.return_value = iter([utils.table.fields.keys()])
rows.export_to_txt(utils.table, temp.name, encoding=self.encoding,
**kwargs)
self.assertTrue(mocked_serialize.called)
self.assertEqual(mocked_serialize.call_count, 1)
call = mocked_serialize.call_args
self.assertEqual(call[0], (utils.table, ))
self.assertEqual(call[1], kwargs)
def __call__(self):
view = getMultiAdapter((self.context, self.request), name='view')
table = view.table()
filename = "%s.txt" % view.filename_prefix()
data = rows.export_to_txt(table)
self.request.response.setHeader('Content-Type', '"%s"' % EXTENSIONS_TYPES.get('text'))
self.request.response.setHeader('Content-Disposition', 'attachment; filename="%s"' % filename)
return data
def print_(input_encoding, output_encoding, input_locale, output_locale,
table_index, verify_ssl, fields, fields_exclude, order_by, source):
import_fields = _get_import_fields(fields, fields_exclude)
# TODO: if create_table implements `fields_exclude` this _import_table call
# will import only the desired data
if input_locale is not None:
with rows.locale_context(input_locale):
table = _import_table(source,
encoding=input_encoding,
verify_ssl=verify_ssl,
index=table_index,
import_fields=import_fields)
else:
table = _import_table(source,
encoding=input_encoding,
verify_ssl=verify_ssl,
index=table_index,
import_fields=import_fields)
if order_by is not None:
order_by = _get_field_names(order_by,
table.field_names,
permit_not=True)
# TODO: use complete list of `order_by` fields
table.order_by(order_by[0].replace('^', '-'))
export_fields = _get_export_fields(table.field_names, fields_exclude)
output_encoding = output_encoding or sys.stdout.encoding or \
DEFAULT_OUTPUT_ENCODING
fobj = BytesIO()
if output_locale is not None:
with rows.locale_context(output_locale):
rows.export_to_txt(table,
fobj,
encoding=output_encoding,
export_fields=export_fields)
else:
rows.export_to_txt(table,
fobj,
encoding=output_encoding,
export_fields=export_fields)
fobj.seek(0)
# TODO: may pass unicode to click.echo if output_encoding is not provided
click.echo(fobj.read())
def _test_import_from_txt_works_with_custom_frame(self, frame_style):
temp = tempfile.NamedTemporaryFile(delete=False)
# self.files_to_delete.append(temp.name)
print("*" * 100, temp.name)
original_data = rows.import_from_txt(self.filename)
rows.export_to_txt(utils.table,
temp.file,
encoding='utf-8',
frame_style=frame_style)
new_data = rows.import_from_txt(temp.name)
self.assertEqual(
list(new_data),
list(original_data),
msg='failed to read information with frame_style == "{0}"'.format(
frame_style))
def _test_import_from_txt_works_with_custom_frame(self, frame_style):
temp = tempfile.NamedTemporaryFile(delete=False)
original_data = rows.import_from_txt(self.filename)
rows.export_to_txt(
utils.table, temp.file, encoding="utf-8", frame_style=frame_style
)
self._reset_txt_plugin()
new_data = rows.import_from_txt(temp.name)
self.assertEqual(
list(new_data),
list(original_data),
msg='failed to read information with frame_style == "{0}"'.format(
frame_style
),
)
def _test_export_to_txt_frame_style(self, frame_style, chars, positive=True):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
rows.export_to_txt(
utils.table, temp.file, encoding="utf-8", frame_style=frame_style
)
if sys.version_info.major < 3:
from io import open as open_
else:
open_ = open
file_data = open_(temp.name, "rt", encoding="utf-8").read()
for char in chars:
if positive:
self.assertIn(char, file_data)
else:
self.assertNotIn(char, file_data)
def needsinfo_query_report(bugs, requestee):
'''Print general Bugzilla query to stdout.'''
bug_table = Table(fields=my_fields)
for bug in bugs:
bug_table.append({
'requestee': requestee,
'weburl': bug.weburl,
'summary': bug.summary[:35],
})
print(export_to_txt(bug_table, export_fields=NEEDSINFO_EXPORT_FIELDS))
def test_export_to_txt_uses_export_data(self, mocked_export_data):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
kwargs = {'encoding': 'utf-8', 'test': 123, 'parameter': 3.14, }
mocked_export_data.return_value = 42
result = rows.export_to_txt(utils.table, temp.name, **kwargs)
self.assertTrue(mocked_export_data.called)
self.assertEqual(mocked_export_data.call_count, 1)
self.assertEqual(result, 42)
call = mocked_export_data.call_args
self.assertEqual(call[0][0], temp.name)
self.assertEqual(call[1], {})
def _test_export_to_txt_frame_style(self,
frame_style,
chars,
positive=True):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
rows.export_to_txt(utils.table,
temp.file,
encoding='utf-8',
frame_style=frame_style)
if sys.version_info.major < 3:
from io import open as open_
else:
open_ = open
file_data = open_(temp.name, 'rt', encoding='utf-8').read()
for char in chars:
if positive:
self.assertIn(char, file_data)
else:
self.assertNotIn(char, file_data)
def test_export_to_txt_uses_export_data(self, mocked_export_data):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
kwargs = {"test": 123, "parameter": 3.14}
mocked_export_data.return_value = 42
result = rows.export_to_txt(
utils.table, temp.name, encoding=self.encoding, **kwargs
)
self.assertTrue(mocked_export_data.called)
self.assertEqual(mocked_export_data.call_count, 1)
self.assertEqual(result, 42)
call = mocked_export_data.call_args
self.assertEqual(call[0][0], temp.name)
self.assertEqual(call[1], {"mode": "wb"})
def test_export_to_txt_uses_export_data(self, mocked_export_data):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
kwargs = {"test": 123, "parameter": 3.14}
mocked_export_data.return_value = 42
result = rows.export_to_txt(utils.table,
temp.name,
encoding=self.encoding,
**kwargs)
self.assertTrue(mocked_export_data.called)
self.assertEqual(mocked_export_data.call_count, 1)
self.assertEqual(result, 42)
call = mocked_export_data.call_args
self.assertEqual(call[0][0], temp.name)
self.assertEqual(call[1], {"mode": "wb"})
def test_export_to_txt_uses_export_data(self, mocked_export_data):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
kwargs = {
'encoding': 'utf-8',
'test': 123,
'parameter': 3.14,
}
mocked_export_data.return_value = 42
result = rows.export_to_txt(utils.table, temp.name, **kwargs)
self.assertTrue(mocked_export_data.called)
self.assertEqual(mocked_export_data.call_count, 1)
self.assertEqual(result, 42)
call = mocked_export_data.call_args
self.assertEqual(call[0][0], temp.name)
self.assertEqual(call[1], {})
def general_query_report(bugs):
'''Print general Bugzilla query to stdout.'''
bug_table = Table(fields=my_fields)
for bug in bugs:
flags = ', '.join([
'{}{}'.format(bug['name'], bug['status']) for bug in bugs[0].flags
if bug['is_active']
])
bug_table.append({
'id': bug.id,
'summary': bug.summary[:25],
'component': bug.component,
'status': bug.status,
'resolution': bug.resolution,
'flags': flags,
'pm_score': bug.cf_pm_score,
})
print(export_to_txt(bug_table, export_fields=GENERAL_EXPORT_FIELDS))
def query(
input_encoding,
output_encoding,
input_locale,
output_locale,
verify_ssl,
samples,
output,
frame_style,
query,
sources,
):
samples = samples if samples > 0 else None
if not query.lower().startswith("select"):
table_names = ", ".join(
["table{}".format(index) for index in range(1, len(sources) + 1)]
)
query = "SELECT * FROM {} WHERE {}".format(table_names, query)
if len(sources) == 1:
source = detect_source(sources[0], verify_ssl=verify_ssl, progress=True)
if source.plugin_name in ("sqlite", "postgresql"):
# Optimization: query the db directly
result = import_from_source(
source, DEFAULT_INPUT_ENCODING, query=query, samples=samples
)
else:
if input_locale is not None:
with rows.locale_context(input_locale):
table = import_from_source(
source, DEFAULT_INPUT_ENCODING, samples=samples
)
else:
table = import_from_source(
source, DEFAULT_INPUT_ENCODING, samples=samples
)
sqlite_connection = sqlite3.Connection(":memory:")
rows.export_to_sqlite(table, sqlite_connection, table_name="table1")
result = rows.import_from_sqlite(sqlite_connection, query=query)
else:
# TODO: if all sources are SQLite we can also optimize the import
if input_locale is not None:
with rows.locale_context(input_locale):
tables = [
_import_table(
source,
encoding=input_encoding,
verify_ssl=verify_ssl,
samples=samples,
)
for source in sources
]
else:
tables = [
_import_table(
source,
encoding=input_encoding,
verify_ssl=verify_ssl,
samples=samples,
)
for source in sources
]
sqlite_connection = sqlite3.Connection(":memory:")
for index, table in enumerate(tables, start=1):
rows.export_to_sqlite(
table, sqlite_connection, table_name="table{}".format(index)
)
result = rows.import_from_sqlite(sqlite_connection, query=query)
# TODO: may use sys.stdout.encoding if output_file = '-'
output_encoding = output_encoding or sys.stdout.encoding or DEFAULT_OUTPUT_ENCODING
if output is None:
fobj = BytesIO()
if output_locale is not None:
with rows.locale_context(output_locale):
rows.export_to_txt(
result, fobj, encoding=output_encoding, frame_style=frame_style
)
else:
rows.export_to_txt(
result, fobj, encoding=output_encoding, frame_style=frame_style
)
fobj.seek(0)
click.echo(fobj.read())
else:
if output_locale is not None:
with rows.locale_context(output_locale):
export_to_uri(result, output, encoding=output_encoding)
else:
export_to_uri(result, output, encoding=output_encoding)
class MyIntegerField(rows.fields.IntegerField):
'''Weird integer represetation, having a `#` just before the number'''
@classmethod
def serialize(cls, value):
return '#' + str(value)
@classmethod
def deserialize(cls, value):
return int(value.replace('#', ''))
class PtBrDateField(rows.fields.DateField):
INPUT_FORMAT = '%d/%m/%Y'
data = [['name', 'age', 'birthdate'],
['alvaro', '#30', '29/04/1987'],
['joao', '#17', '01/02/2000']]
table = rows.plugins.utils.create_table(
data,
force_types={'age': MyIntegerField,
'birthdate': PtBrDateField,})
print(type(table[0].age)) # `<class 'int'>`
print(type(table[0].birthdate)) # `<class 'datetime.date'>`
print(rows.export_to_txt(table)) # "age" values will start with "#"
def print_(input_encoding, output_encoding, input_locale, output_locale,
table_index, verify_ssl, fields, fields_except, order_by, source):
if fields is not None and fields_except is not None:
click.echo('ERROR: `--fields` cannot be used with `--fields-except`',
err=True)
sys.exit(20)
# TODO: may use sys.stdout.encoding if output_file = '-'
output_encoding = output_encoding or sys.stdout.encoding or \
DEFAULT_OUTPUT_ENCODING
# TODO: may use `import_fields` for better performance
if input_locale is not None:
with rows.locale_context(input_locale):
table = _import_table(source,
encoding=input_encoding,
verify_ssl=verify_ssl,
index=table_index)
else:
table = _import_table(source,
encoding=input_encoding,
verify_ssl=verify_ssl,
index=table_index)
table_field_names = table.fields.keys()
if fields is not None:
fields = _get_field_names(fields, table_field_names)
if fields_except is not None:
fields_except = _get_field_names(fields_except, table_field_names)
if fields is not None and fields_except is None:
export_fields = fields
elif fields is not None and fields_except is not None:
export_fields = list(fields)
map(export_fields.remove, fields_except)
elif fields is None and fields_except is not None:
export_fields = list(table_field_names)
map(export_fields.remove, fields_except)
else:
export_fields = table_field_names
if order_by is not None:
order_by = _get_field_names(order_by,
table_field_names,
permit_not=True)
# TODO: use complete list of `order_by` fields
table.order_by(order_by[0].replace('^', '-'))
fobj = BytesIO()
if output_locale is not None:
with rows.locale_context(output_locale):
rows.export_to_txt(table,
fobj,
encoding=output_encoding,
export_fields=export_fields)
else:
rows.export_to_txt(table,
fobj,
encoding=output_encoding,
export_fields=export_fields)
fobj.seek(0)
# TODO: may pass unicode to click.echo if output_encoding is not provided
click.echo(fobj.read())
def query(input_encoding, output_encoding, input_locale, output_locale,
verify_ssl, fields, output, query, sources):
# TODO: may use sys.stdout.encoding if output_file = '-'
output_encoding = output_encoding or sys.stdout.encoding or \
DEFAULT_OUTPUT_ENCODING
if not query.lower().startswith('select'):
field_names = '*' if fields is None else fields
table_names = ', '.join(
['table{}'.format(index) for index in range(1,
len(sources) + 1)])
query = 'SELECT {} FROM {} WHERE {}'.format(field_names, table_names,
query)
if len(sources) == 1:
source = detect_source(sources[0], verify_ssl=verify_ssl)
if source.plugin_name != 'sqlite':
if input_locale is not None:
with rows.locale_context(input_locale):
table = import_from_source(source, DEFAULT_INPUT_ENCODING)
else:
table = import_from_source(source, DEFAULT_INPUT_ENCODING)
sqlite_connection = sqlite3.Connection(':memory:')
rows.export_to_sqlite(table,
sqlite_connection,
table_name='table1')
result = rows.import_from_sqlite(sqlite_connection, query=query)
else:
# Optimization: query the SQLite database directly
result = import_from_source(source,
DEFAULT_INPUT_ENCODING,
query=query)
else:
if input_locale is not None:
with rows.locale_context(input_locale):
tables = [
_import_table(source,
encoding=input_encoding,
verify_ssl=verify_ssl) for source in sources
]
else:
tables = [
_import_table(source,
encoding=input_encoding,
verify_ssl=verify_ssl) for source in sources
]
sqlite_connection = sqlite3.Connection(':memory:')
for index, table in enumerate(tables, start=1):
rows.export_to_sqlite(table,
sqlite_connection,
table_name='table{}'.format(index))
result = rows.import_from_sqlite(sqlite_connection, query=query)
if output is None:
fobj = BytesIO()
if output_locale is not None:
with rows.locale_context(output_locale):
rows.export_to_txt(result, fobj, encoding=output_encoding)
else:
rows.export_to_txt(result, fobj, encoding=output_encoding)
fobj.seek(0)
click.echo(fobj.read())
else:
if output_locale is not None:
with rows.locale_context(output_locale):
export_to_uri(result, output, encoding=output_encoding)
else:
export_to_uri(result, output, encoding=output_encoding)
def test_export_to_txt_filename(self):
temp = tempfile.NamedTemporaryFile(delete=False)
self.files_to_delete.append(temp.name)
rows.export_to_txt(utils.table, temp.name)
self.assert_file_contents_equal(temp.name, self.filename)
def test_export_to_text_should_return_unicode(self):
result = rows.export_to_txt(utils.table)
self.assertEqual(type(result), six.text_type)
import io
import requests
import rows
url = "http://balneabilidade.inema.ba.gov.br/index.php/relatoriodebalneabilidade/geraBoletim?idcampanha=42041"
print("*** Downloading PDF...")
response = requests.get(url)
# The line below will automatically identify the table in all PDF pages - it
# works for this file but not for all cases. You can be more specific defining
# the page numbers, a start/end string (like the header/footer strings) and
# also change the table identification algorithm. Check `backend`, `algorithm`,
# `starts_after`, `ends_before` and `page_numbers` parameters.
# For this simple case you could also install rows' CLI (`pip install
# rows[cli]`) and run: `rows print <url>`
table = rows.import_from_pdf(io.BytesIO(response.content))
rows.export_to_csv(table, "beach-data.csv")
print("*** Table exported to beach-data.csv")
print("*** Extracted table:")
print(rows.export_to_txt(table))
# You could also iterate over the object, like:
# for row in table: print(row)
print("\n\n*** Extracted text:")
text_pages = rows.plugins.pdf.pdf_to_text(io.BytesIO(response.content))
print("\n\n".join(text_pages))
def test_export_to_text_should_return_unicode(self):
result = rows.export_to_txt(utils.table)
self.assertEqual(type(result), six.text_type)
def print_(input_encoding, output_encoding, input_locale, output_locale,
table_index, verify_ssl, fields, fields_except, order_by, source):
if fields is not None and fields_except is not None:
click.echo('ERROR: `--fields` cannot be used with `--fields-except`',
err=True)
sys.exit(20)
output_encoding = output_encoding or sys.stdout.encoding or \
DEFAULT_OUTPUT_ENCODING
# TODO: may use `import_fields` for better performance
if input_locale is not None:
with rows.locale_context(input_locale):
table = _import_table(source, encoding=input_encoding,
verify_ssl=verify_ssl,
index=table_index)
else:
table = _import_table(source, encoding=input_encoding,
verify_ssl=verify_ssl,
index=table_index)
table_field_names = table.field_names
if fields is not None:
fields = _get_field_names(fields, table_field_names)
if fields_except is not None:
fields_except = _get_field_names(fields_except, table_field_names)
# TODO: should set `export_fields = None` if `--fields` and
# `--fields-except` are `None`
if fields is not None and fields_except is None:
export_fields = fields
elif fields is not None and fields_except is not None:
export_fields = list(fields)
for field_to_remove in fields_except:
export_fields.remove(field_to_remove)
elif fields is None and fields_except is not None:
export_fields = list(table_field_names)
for field_to_remove in fields_except:
export_fields.remove(field_to_remove)
else:
export_fields = table_field_names
if order_by is not None:
order_by = _get_field_names(order_by,
table_field_names,
permit_not=True)
# TODO: use complete list of `order_by` fields
table.order_by(order_by[0].replace('^', '-'))
fobj = BytesIO()
if output_locale is not None:
with rows.locale_context(output_locale):
rows.export_to_txt(table, fobj, encoding=output_encoding,
export_fields=export_fields)
else:
rows.export_to_txt(table, fobj, encoding=output_encoding,
export_fields=export_fields)
fobj.seek(0)
# TODO: may pass unicode to click.echo if output_encoding is not provided
click.echo(fobj.read())
import requests
import rows
url = "http://balneabilidade.inema.ba.gov.br/index.php/relatoriodebalneabilidade/geraBoletim?idcampanha=42041"
print("*** Downloading PDF...")
response = requests.get(url)
# The line below will automatically identify the table in all PDF pages - it
# works for this file but not for all cases. You can be more specific defining
# the page numbers, a start/end string (like the header/footer strings) and
# also change the table identification algorithm. Check `backend`, `algorithm`,
# `starts_after`, `ends_before` and `page_numbers` parameters.
# For this simple case you could also install rows' CLI (`pip install
# rows[cli]`) and run: `rows print <url>`
table = rows.import_from_pdf(io.BytesIO(response.content))
rows.export_to_csv(table, "beach-data.csv")
print("*** Table exported to beach-data.csv")
print("*** Extracted table:")
print(rows.export_to_txt(table))
# You could also iterate over the object, like:
# for row in table: print(row)
print("\n\n*** Extracted text:")
text_pages = rows.plugins.pdf.pdf_to_text(io.BytesIO(response.content))
print("\n\n".join(text_pages))
def search_species(name):
'Search species and return a `rows.Table` with results'
response = requests.get(URL, params={'name': name, 'format': 'json'})
json_result = response.json()
if 'error_message' in json_result and \
json_result['error_message'].strip() != '':
raise RuntimeError(json_result['error_message'])
# Need to dump again to JSON (to get a `bytes` object) and then import as
# JSON, since there is a little bug on rows
json_rows = json.dumps(json_result['results'])
return rows.import_from_json(BytesIO(json_rows))
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('species_name')
parser.add_argument('--output')
args = parser.parse_args()
result = search_species(args.species_name)
if args.output is None:
print(rows.export_to_txt(result))
else:
rows.utils.export_to_uri(result, args.output)
class MyIntegerField(rows.fields.IntegerField):
"""Weird integer represetation, having a `#` just before the number"""
@classmethod
def serialize(cls, value):
return "#" + str(value)
@classmethod
def deserialize(cls, value):
return int(value.replace("#", ""))
class PtBrDateField(rows.fields.DateField):
INPUT_FORMAT = "%d/%m/%Y"
data = [
["name", "age", "birthdate"],
["alvaro", "#30", "29/04/1987"],
["joao", "#17", "01/02/2000"],
]
table = rows.plugins.utils.create_table(data,
force_types={
"age": MyIntegerField,
"birthdate": PtBrDateField
})
print(type(table[0].age)) # `<class 'int'>`
print(type(table[0].birthdate)) # `<class 'datetime.date'>`
print(rows.export_to_txt(table)) # "age" values will start with "#"
def query(input_encoding, output_encoding, input_locale, output_locale,
verify_ssl, fields, output, query, sources):
# TODO: may use sys.stdout.encoding if output_file = '-'
output_encoding = output_encoding or sys.stdout.encoding or \
DEFAULT_OUTPUT_ENCODING
if not query.lower().startswith('select'):
field_names = '*' if fields is None else fields
table_names = ', '.join(['table{}'.format(index)
for index in range(1, len(sources) + 1)])
query = 'SELECT {} FROM {} WHERE {}'.format(field_names, table_names,
query)
if len(sources) == 1:
source = detect_source(sources[0], verify_ssl=verify_ssl)
if source.plugin_name != 'sqlite':
if input_locale is not None:
with rows.locale_context(input_locale):
table = import_from_source(source, DEFAULT_INPUT_ENCODING)
else:
table = import_from_source(source, DEFAULT_INPUT_ENCODING)
sqlite_connection = sqlite3.Connection(':memory:')
rows.export_to_sqlite(table,
sqlite_connection,
table_name='table1')
result = rows.import_from_sqlite(sqlite_connection, query=query)
else:
# Optimization: query the SQLite database directly
result = import_from_source(source,
DEFAULT_INPUT_ENCODING,
query=query)
else:
if input_locale is not None:
with rows.locale_context(input_locale):
tables = [_import_table(source, encoding=input_encoding,
verify_ssl=verify_ssl)
for source in sources]
else:
tables = [_import_table(source, encoding=input_encoding,
verify_ssl=verify_ssl)
for source in sources]
sqlite_connection = sqlite3.Connection(':memory:')
for index, table in enumerate(tables, start=1):
rows.export_to_sqlite(table,
sqlite_connection,
table_name='table{}'.format(index))
result = rows.import_from_sqlite(sqlite_connection, query=query)
if output is None:
fobj = BytesIO()
if output_locale is not None:
with rows.locale_context(output_locale):
rows.export_to_txt(result, fobj, encoding=output_encoding)
else:
rows.export_to_txt(result, fobj, encoding=output_encoding)
fobj.seek(0)
click.echo(fobj.read())
else:
if output_locale is not None:
with rows.locale_context(output_locale):
export_to_uri(result, output, encoding=output_encoding)
else:
export_to_uri(result, output, encoding=output_encoding)
def _create_response(result, output):
if not output:
rows.export_to_txt(result, sys.stdout)
else:
rows.utils.export_to_uri(output, result)
