def description(self):
if self._description:
return self._description
if not self.has_result_set:
return None
self._description = [(
self._meta_data.getColumnName(i),
self._converter.get_jdbc_type_name(
self._meta_data.getColumnType(i)),
self._meta_data.getColumnDisplaySize(i),
None,
self._meta_data.getPrecision(i),
self._meta_data.getScale(i),
self._meta_data.isNullable(i),
) for i in xrange(1,
self._meta_data.getColumnCount() + 1)]
return self._description
def _process_rows(self, response):
result_set = response.get('ResultSet', None)
if not result_set:
raise DataError('KeyError `ResultSet`')
rows = result_set.get('Rows', None)
if rows is None:
raise DataError('KeyError `Rows`')
processed_rows = []
if len(rows) > 0:
offset = 1 if not self._next_token and self._is_first_row_column_labels(rows) else 0
processed_rows = [
tuple([self._converter.convert(meta.get('Type', None),
row.get('VarCharValue', None))
for meta, row in zip(self._meta_data, rows[i].get('Data', []))])
for i in xrange(offset, len(rows))
]
self._rows.extend(processed_rows)
self._next_token = response.get('NextToken', None)
def test_empty_result(self, cursor):
table = "test_pandas_cursor_empty_result_" + "".join(
[random.choice(string.ascii_lowercase + string.digits) for _ in xrange(10)]
)
location = "{0}{1}/{2}/".format(ENV.s3_staging_dir, S3_PREFIX, table)
query_id, future = cursor.execute(
"""
CREATE EXTERNAL TABLE IF NOT EXISTS
{schema}.{table} (number_of_rows INT)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
LINES TERMINATED BY '\n' STORED AS TEXTFILE
LOCATION '{location}'
""".format(
schema=SCHEMA, table=table, location=location
)
)
df = future.result().as_pandas()
self.assertEqual(df.shape[0], 0)
self.assertEqual(df.shape[1], 0)
def _process_rows(self, response):
result_set = response.get("ResultSet", None)
if not result_set:
raise DataError("KeyError `ResultSet`")
rows = result_set.get("Rows", None)
if rows is None:
raise DataError("KeyError `Rows`")
processed_rows = []
if len(rows) > 0:
offset = (1 if not self._next_token
and self._is_first_row_column_labels(rows) else 0)
processed_rows = [
tuple([
self._converter.convert(meta.get("Type", None),
row.get("VarCharValue", None))
for meta, row in zip(self._meta_data, rows[i].get(
"Data", []))
]) for i in xrange(offset, len(rows))
]
self._rows.extend(processed_rows)
self._next_token = response.get("NextToken", None)
def test_fetchall(self, cursor):
cursor.execute('SELECT * FROM one_row')
self.assertEqual(cursor.fetchall(), [(1,)])
cursor.execute('SELECT a FROM many_rows ORDER BY a')
self.assertEqual(cursor.fetchall(), [(i,) for i in xrange(10000)])
def test_null(self, cursor):
cursor.execute('SELECT null FROM many_rows')
self.assertEqual(cursor.fetchall(), [(None,)] * 10000)
cursor.execute('SELECT IF(a %% 11 = 0, null, a) FROM many_rows')
self.assertEqual(cursor.fetchall(),
[(None if a % 11 == 0 else a,) for a in xrange(10000)])
def test_many_as_pandas(self, cursor):
df = cursor.execute("SELECT * FROM many_rows").as_pandas()
self.assertEqual(df.shape[0], 10000)
self.assertEqual(df.shape[1], 1)
self.assertEqual([(row["a"], ) for _, row in df.iterrows()],
[(i, ) for i in xrange(10000)])
import contextlib
import os
import random
import string
import pytest
from past.builtins.misc import xrange
from pyathena import connect
from tests.util import Env, read_query
ENV = Env()
BASE_PATH = os.path.dirname(os.path.abspath(__file__))
S3_PREFIX = 'test_pyathena'
SCHEMA = 'test_pyathena_' + ''.join([
random.choice(string.ascii_lowercase + string.digits) for i in xrange(10)
])
@pytest.fixture(scope='session', autouse=True)
def _setup_session(request):
request.addfinalizer(_teardown_session)
with contextlib.closing(connect()) as conn:
with conn.cursor() as cursor:
_create_database(cursor)
_create_table(cursor)
def _teardown_session():
with contextlib.closing(connect()) as conn:
with conn.cursor() as cursor:
import contextlib
import os
import random
import string
import pytest
from past.builtins.misc import xrange
from pyathena import connect
from tests.util import Env, read_query
ENV = Env()
BASE_PATH = os.path.dirname(os.path.abspath(__file__))
S3_PREFIX = 'test_pyathena'
SCHEMA = 'test_pyathena_' + ''.join([random.choice(
string.ascii_lowercase + string.digits) for _ in xrange(10)])
@pytest.fixture(scope='session', autouse=True)
def _setup_session(request):
request.addfinalizer(_teardown_session)
with contextlib.closing(connect()) as conn:
with conn.cursor() as cursor:
_create_database(cursor)
_create_table(cursor)
def _teardown_session():
with contextlib.closing(connect()) as conn:
with conn.cursor() as cursor:
_drop_database(cursor)
