def download(self, notebook, snippet, format):
try:
db = self._get_db(snippet)
handle = self._get_handle(snippet)
# Test handle to verify if still valid
db.fetch(handle, start_over=True, rows=1)
if notebook.get('name'):
file_name = '%(name)s' % notebook
else:
file_name = '%(type)s-%(id)s' % notebook
return data_export.download(handle, format, db, id=snippet['id'], file_name=file_name)
except Exception, e:
title = 'The query result cannot be downloaded.'
LOG.exception(title)
if hasattr(e, 'message') and e.message:
if 'generic failure: Unable to find a callback: 32775' in e.message:
message = e.message + " " + _("Increase the sasl_max_buffer value in hue.ini")
else:
message = e.message
else:
message = e
raise PopupException(_(title), detail=message)
def test_data_export(self):
hql = 'SELECT * FROM test'
query = hql_query(hql)
# Get the result in xls.
handle = self.db.execute_and_wait(query)
xls_resp = download(handle, 'xls', self.db)
translated_csv = xls2csv(xls_resp.content)
# It should have 257 lines (256 + header)
assert_equal(len(translated_csv.strip('\r\n').split('\r\n')), 257)
# Get the result in csv.
query = hql_query(hql)
handle = self.db.execute_and_wait(query)
csv_resp = download(handle, 'csv', self.db)
assert_equal(csv_resp.content, translated_csv)
def test_data_export(self):
hql = 'SELECT * FROM test'
query = hql_query(hql)
# Get the result in xls.
handle = self.db.execute_and_wait(query)
xls_resp = download(handle, 'xls', self.db)
# Should be CSV since we simply change the file extension and MIME type from CSV to XLS.
translated_csv = xls_resp.content
# It should have 257 lines (256 + header)
assert_equal(len(translated_csv.strip('\r\n').split('\r\n')), 257)
# Get the result in csv.
query = hql_query(hql)
handle = self.db.execute_and_wait(query)
csv_resp = download(handle, 'csv', self.db)
assert_equal(csv_resp.content, translated_csv)
def download(request, id, format):
id = int(id)
assert format in common.DL_FORMATS
query_history = models.QueryHistory.objects.get(id=id)
LOG.debug('Download results for query %s: [ %s ]' %
(query_history.server_id, query_history.query))
return data_export.download(query_history, format)
def download(request, id, format):
assert format in common.DL_FORMATS
query_history = authorized_get_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server())
LOG.debug('Download results for query %s: [ %s ]' % (query_history.server_id, query_history.query))
return data_export.download(query_history.get_handle(), format, db)
def test_data_export_limit_clause(self):
limit = 3
hql = 'SELECT foo FROM test limit %d' % (limit,)
query = hql_query(hql)
handle = self.db.execute_and_wait(query)
# Get the result in csv. Should have 3 + 1 header row.
csv_resp = download(handle, 'csv', self.db)
assert_equal(len(csv_resp.content.strip().split('\n')), limit + 1)
def download(request, id, format):
id = int(id)
assert format in common.DL_FORMATS
query_history = authorized_get_history(request, id, must_exist=True)
LOG.debug('Download results for query %s: [ %s ]' %
(query_history.server_id, query_history.query))
return data_export.download(query_history, format)
def download(self, notebook, snippet, format):
try:
db = self._get_db(snippet)
handle = self._get_handle(snippet)
return data_export.download(handle, format, db)
except Exception, e:
if not hasattr(e, 'message') or not e.message:
message = e
else:
message = e.message
raise PopupException(message, detail='')
def download(request, id, format):
try:
query_history = authorized_get_query_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
LOG.debug('Download results for query %s: [ %s ]' % (query_history.server_id, query_history.query))
return data_export.download(query_history.get_handle(), format, db)
except Exception, e:
if not hasattr(e, 'message') or not e.message:
message = e
else:
message = e.message
raise PopupException(message, detail='')
def test_data_download(self):
hql = 'SELECT * FROM tweets %(limit)s'
FETCH_SIZE = data_export.FETCH_SIZE
data_export.FETCH_SIZE = 2 # Decrease fetch size to validate last fetch logic
try:
query = hql_query(hql % {'limit': ''})
handle = self.db.execute_and_wait(query)
# Get the result in csv. Should have 5 + 1 header row.
csv_resp = download(handle, 'csv', self.db)
csv_content = ''.join(csv_resp.streaming_content)
assert_equal(len(csv_content.strip().split('\n')), 5 + 1)
query = hql_query(hql % {'limit': 'LIMIT 0'})
handle = self.db.execute_and_wait(query)
csv_resp = download(handle, 'csv', self.db)
csv_content = ''.join(csv_resp.streaming_content)
assert_equal(len(csv_content.strip().split('\n')), 1)
query = hql_query(hql % {'limit': 'LIMIT 1'})
handle = self.db.execute_and_wait(query)
csv_resp = download(handle, 'csv', self.db)
csv_content = ''.join(csv_resp.streaming_content)
assert_equal(len(csv_content.strip().split('\n')), 1 + 1)
query = hql_query(hql % {'limit': 'LIMIT 2'})
handle = self.db.execute_and_wait(query)
csv_resp = download(handle, 'csv', self.db)
csv_content = ''.join(csv_resp.streaming_content)
assert_equal(len(csv_content.strip().split('\n')), 1 + 2)
finally:
data_export.FETCH_SIZE = FETCH_SIZE
def download(self, notebook, snippet, format):
try:
db = self._get_db(snippet)
handle = self._get_handle(snippet)
# Test handle to verify still valid
db.get_state(handle)
return data_export.download(handle, format, db)
except Exception, e:
LOG.exception('error downloading notebook')
if not hasattr(e, 'message') or not e.message:
message = e
else:
message = e.message
raise PopupException(message, detail='')
def download(self, notebook, snippet, format):
try:
db = self._get_db(snippet)
handle = self._get_handle(snippet)
# Test handle to verify if still valid
db.fetch(handle, start_over=True, rows=1)
return data_export.download(handle, format, db, id=snippet['id'])
except Exception, e:
title = 'The query result cannot be downloaded.'
LOG.exception(title)
if hasattr(e, 'message') and e.message:
message = e.message
else:
message = e
raise PopupException(_(title), detail=message)
def download(request, id, format, user_agent=None):
if not ENABLE_DOWNLOAD.get():
return serve_403_error(request)
try:
query_history = authorized_get_query_history(request, id, must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
LOG.debug('Download results for query %s: [ %s ]' % (query_history.server_id, query_history.query))
return data_export.download(query_history.get_handle(), format, db, user_agent=user_agent)
except Exception as e:
if not hasattr(e, 'message') or not e.message:
message = e
else:
message = e.message
raise PopupException(message, detail='')
def download(self, notebook, snippet, format, user_agent=None):
file_name = _get_snippet_name(notebook)
guid = uuid.uuid4().hex
connection = self.engine.connect()
result = connection.execute(snippet['statement'])
CONNECTION_CACHE[guid] = {
'connection': connection,
'result': result
}
db = FixedResult([col[0] if type(col) is dict or type(col) is tuple else col for col in result.cursor.description])
def callback():
connection = CONNECTION_CACHE.get(guid)
if connection:
connection['connection'].close()
del CONNECTION_CACHE[guid]
return data_export.download({'guid': guid}, format, db, id=snippet['id'], file_name=file_name, callback=callback)
def download(request, id, format):
try:
query_history = authorized_get_query_history(request,
id,
must_exist=True)
db = dbms.get(request.user, query_history.get_query_server_config())
LOG.debug('Download results for query %s: [ %s ]' %
(query_history.server_id, query_history.query))
return data_export.download(query_history.get_handle(), format, db)
except Exception, e:
if not hasattr(e, 'message') or not e.message:
message = e
else:
message = e.message
raise PopupException(message, detail='')
def download(self, notebook, snippet, format):
try:
db = self._get_db(snippet)
handle = self._get_handle(snippet)
# Test handle to verify if still valid
db.fetch(handle, start_over=True, rows=1)
return data_export.download(handle, format, db, id=snippet['id'])
except Exception, e:
title = 'The query result cannot be downloaded.'
LOG.exception(title)
if hasattr(e, 'message') and e.message:
message = e.message
else:
message = e
raise PopupException(_(title), detail=message)
def download(self,
notebook,
snippet,
format,
user_agent=None,
max_rows=None,
store_data_type_in_header=False):
file_name = _get_snippet_name(notebook)
results = self._execute(notebook, snippet)
db = FixedResult(results)
return data_export.download(None,
format,
db,
id=snippet['id'],
file_name=file_name,
user_agent=user_agent)
def download(self, notebook, snippet, format):
try:
db = self._get_db(snippet)
handle = self._get_handle(snippet)
# Test handle to verify if still valid
db.fetch(handle, start_over=True, rows=1)
file_name = _get_snippet_name(notebook)
return data_export.download(handle, format, db, id=snippet['id'], file_name=file_name)
except Exception, e:
title = 'The query result cannot be downloaded.'
LOG.exception(title)
if hasattr(e, 'message') and e.message:
if 'generic failure: Unable to find a callback: 32775' in e.message:
message = e.message + " " + _("Increase the sasl_max_buffer value in hue.ini")
elif 'query result cache exceeded its limit' in e.message:
message = e.message.replace("Restarting the fetch is not possible.", _("Please execute the query again."))
else:
message = e.message
else:
message = e
raise PopupException(_(title), detail=message)
