def test_show_tables_hs2(self):
grant_access("test", "default", "metastore")
with patch('beeswax.server.dbms.get') as get:
with patch('beeswax.server.dbms.KazooClient') as KazooClient:
server_config = get_query_server_config(name='beeswax')
KazooClient.return_value = Mock(
# Bug "TypeError: expected string or buffer" if False, to add a new test case and fix
exists=Mock(return_value=True),
get_children=Mock(
return_value=['serverUri=hive-llap-1.gethue.com:10000;serverUri=hive-llap-2.gethue.com:10000'])
)
get.return_value = Mock(
get_databases=Mock(
return_value=['sfdc']
),
get_database=Mock(
return_value={}
),
get_tables_meta=Mock(
return_value=[{'name': 'customer'}, {'name': 'opportunities'}]
),
server_name='hive'
)
response = self.client.post('/metastore/tables/sfdc?format=json')
get.assert_called()
assert_equal(response.status_code, 200)
data = json.loads(response.content)
assert_equal(data['status'], 0)
assert_equal(data['table_names'], ['customer', 'opportunities'])
assert_equal(data['tables'], [{'name': 'customer'}, {'name': 'opportunities'}])
def setup_class(cls):
cls.finish = []
if not is_live_cluster():
raise SkipTest
cls.client = make_logged_in_client()
cls.user = User.objects.get(username='******')
add_to_group('test')
cls.db = dbms.get(cls.user, get_query_server_config(name='impala'))
cls.DATABASE = get_db_prefix(name='impala')
queries = [
'DROP TABLE IF EXISTS %(db)s.tweets;' % {
'db': cls.DATABASE
},
'DROP DATABASE IF EXISTS %(db)s CASCADE;' % {
'db': cls.DATABASE
},
'CREATE DATABASE %(db)s;' % {
'db': cls.DATABASE
}
]
for query in queries:
resp = _make_query(cls.client,
query,
database='default',
local=False,
server_name='impala')
resp = wait_for_query_to_finish(cls.client, resp, max=180.0)
content = json.loads(resp.content)
assert_true(content['status'] == 0, resp.content)
queries = [
"""
CREATE TABLE tweets (row_num INTEGER, id_str STRING, text STRING) STORED AS PARQUET;
""", """
INSERT INTO TABLE tweets VALUES (1, "531091827395682000", "My dad looks younger than costa");
""", """
INSERT INTO TABLE tweets VALUES (2, "531091827781550000", "There is a thin line between your partner being vengeful and you reaping the consequences of your bad actions towards your partner.");
""", """
INSERT INTO TABLE tweets VALUES (3, "531091827768979000", "@Mustang_Sally83 and they need to get into you :))))");
""", """
INSERT INTO TABLE tweets VALUES (4, "531091827114668000", "@RachelZJohnson thank you rach!xxx");
""", """
INSERT INTO TABLE tweets VALUES (5, "531091827949309000", "i think @WWERollins was robbed of the IC title match this week on RAW also i wonder if he will get a rematch i hope so @WWE");
"""
]
for query in queries:
resp = _make_query(cls.client,
query,
database=cls.DATABASE,
local=False,
server_name='impala')
resp = wait_for_query_to_finish(cls.client, resp, max=180.0)
content = json.loads(resp.content)
assert_true(content['status'] == 0, resp.content)
def setUp(self):
self.client.post('/beeswax/install_examples')
self.user = User.objects.get(username='******')
grant_access("test", "test", "notebook")
self.db = dbms.get(self.user, get_query_server_config())
self.cluster.fs.do_as_user('test', self.cluster.fs.create_home_dir,
'/user/test')
self.api = HS2Api(self.user)
self.notebook_json = """
{
"uuid": "f5d6394d-364f-56e8-6dd3-b1c5a4738c52",
"id": 1234,
"sessions": [{"type": "hive", "properties": [], "id": "1234"}],
"type": "query-hive",
"name": "Test Hiveserver2 Editor",
"isSaved": false,
"parentUuid": null
}
"""
self.statement = 'SELECT description, salary FROM sample_07 WHERE (sample_07.salary > 100000) ORDER BY salary DESC LIMIT 1000'
self.snippet_json = """
{
"status": "running",
"database": "%(database)s",
"id": "d70d31ee-a62a-4854-b2b1-b852f6a390f5",
"result": {
"type": "table",
"handle": {},
"id": "ca11fcb1-11a5-f534-8200-050c8e1e57e3"
},
"statement": "%(statement)s",
"statement_raw": "%(statement)s",
"type": "hive",
"properties": {
"files": [],
"functions": [],
"settings": [
{
"value": "mr",
"key": "hive.execution.engine"
}
]
}
}
""" % {
'database': self.db_name,
'statement': self.statement
}
doc, created = Document2.objects.get_or_create(id=1234,
name='Test Hive Query',
type='query-hive',
owner=self.user,
is_history=True,
data=self.notebook_json)
def setUp(self):
self.client = make_logged_in_client(username="******", groupname="test", recreate=False, is_superuser=False)
self.user = User.objects.get(username='******')
add_to_group('test')
grant_access("test", "test", "notebook")
self.db = dbms.get(self.user, get_query_server_config())
self.api = HS2Api(self.user)
def setUp(self):
self.client = make_logged_in_client(username="******", groupname="test", recreate=False, is_superuser=False)
self.user = User.objects.get(username='******')
add_to_group('test')
grant_access("test", "test", "notebook")
self.db = dbms.get(self.user, get_query_server_config())
self.api = HS2Api(self.user)
def setUp(self):
self.finish = []
# We need a real Impala cluster currently
if not 'impala' in sys.argv and not os.environ.get(
'TEST_IMPALAD_HOST'):
raise SkipTest
if os.environ.get('TEST_IMPALAD_HOST'):
self.finish.append(
SERVER_HOST.set_for_testing(
os.environ.get('TEST_IMPALAD_HOST')))
self.client = make_logged_in_client()
self.user = User.objects.get(username='******')
add_to_group('test')
self.db = dbms.get(self.user, get_query_server_config(name='impala'))
hql = """
USE default;
DROP TABLE IF EXISTS %(db)s.tweets;
DROP DATABASE IF EXISTS %(db)s;
CREATE DATABASE %(db)s;
USE %(db)s;
""" % {
'db': self.DATABASE
}
resp = _make_query(self.client,
hql,
database='default',
local=False,
server_name='impala')
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
hql = """
CREATE TABLE tweets (row_num INTEGER, id_str STRING, text STRING) STORED AS PARQUET;
INSERT INTO TABLE tweets VALUES (1, "531091827395682000", "My dad looks younger than costa");
INSERT INTO TABLE tweets VALUES (2, "531091827781550000", "There is a thin line between your partner being vengeful and you reaping the consequences of your bad actions towards your partner.");
INSERT INTO TABLE tweets VALUES (3, "531091827768979000", "@Mustang_Sally83 and they need to get into you :))))");
INSERT INTO TABLE tweets VALUES (4, "531091827114668000", "@RachelZJohnson thank you rach!xxx");
INSERT INTO TABLE tweets VALUES (5, "531091827949309000", "i think @WWERollins was robbed of the IC title match this week on RAW also i wonder if he will get a rematch i hope so @WWE");
"""
resp = _make_query(self.client,
hql,
database=self.DATABASE,
local=False,
server_name='impala')
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
def tearDown(self):
for f in self.finish:
f()
def setUp(self):
self.client.post('/beeswax/install_examples')
self.user = User.objects.get(username='******')
add_to_group('test')
grant_access("test", "test", "notebook")
self.db = dbms.get(self.user, get_query_server_config())
self.cluster.fs.do_as_user('test', self.cluster.fs.create_home_dir, '/user/test')
self.api = HS2Api(self.user)
self.statement = 'SELECT description, salary FROM sample_07 WHERE (sample_07.salary > 100000) ORDER BY salary DESC LIMIT 1000'
def setUp(self):
self.client.post('/beeswax/install_examples')
self.user = User.objects.get(username='******')
grant_access("test", "test", "notebook")
self.db = dbms.get(self.user, get_query_server_config())
self.cluster.fs.do_as_user('test', self.cluster.fs.create_home_dir,
'/user/test')
self.api = HS2Api(self.user)
self.statement = 'SELECT description, salary FROM sample_07 WHERE (sample_07.salary > 100000) ORDER BY salary DESC LIMIT 1000'
def setUp(self):
self.client.post('/beeswax/install_examples')
self.user = User.objects.get(username='******')
grant_access("test", "test", "notebook")
self.db = dbms.get(self.user, get_query_server_config())
self.cluster.fs.do_as_user('test', self.cluster.fs.create_home_dir, '/user/test')
self.api = HS2Api(self.user)
self.notebook_json = """
{
"uuid": "f5d6394d-364f-56e8-6dd3-b1c5a4738c52",
"id": 1234,
"sessions": [{"type": "hive", "properties": [], "id": "1234"}],
"type": "query-hive",
"name": "Test Hiveserver2 Editor",
"isSaved": false,
"parentUuid": null
}
"""
self.statement = 'SELECT description, salary FROM sample_07 WHERE (sample_07.salary > 100000) ORDER BY salary DESC LIMIT 1000'
self.snippet_json = """
{
"status": "running",
"database": "%(database)s",
"id": "d70d31ee-a62a-4854-b2b1-b852f6a390f5",
"result": {
"type": "table",
"handle": {},
"id": "ca11fcb1-11a5-f534-8200-050c8e1e57e3"
},
"statement": "%(statement)s",
"statement_raw": "%(statement)s",
"type": "hive",
"properties": {
"files": [],
"functions": [],
"settings": []
}
}
""" % {'database': self.db_name, 'statement': self.statement}
doc, created = Document2.objects.get_or_create(
id=1234,
name='Test Hive Query',
type='query-hive',
owner=self.user,
is_history=True,
data=self.notebook_json)
def test_thrift_over_http_config():
resets = [
conf.SERVER_HOST.set_for_testing('impalad_host'),
conf.SERVER_PORT.set_for_testing(21050),
conf.USE_THRIFT_HTTP.set_for_testing(True)
]
with patch('impala.dbms.get_hs2_http_port') as get_hs2_http_port:
get_hs2_http_port.return_value = 30000
try:
query_server = get_query_server_config(name='impala')
assert_equal(query_server['server_port'], 30000)
assert_equal(query_server['transport_mode'], 'http')
assert_equal(query_server['http_url'], 'http://impalad_host:30000')
finally:
for reset in resets:
reset()
def setup_class(cls):
cls.finish = []
if not is_live_cluster():
raise SkipTest
cls.client = make_logged_in_client()
cls.user = User.objects.get(username='******')
add_to_group('test')
cls.db = dbms.get(cls.user, get_query_server_config(name='impala'))
cls.DATABASE = get_db_prefix(name='impala')
queries = ["""
DROP TABLE IF EXISTS %(db)s.tweets;
""" % {'db': cls.DATABASE}, """
DROP DATABASE IF EXISTS %(db)s CASCADE;
""" % {'db': cls.DATABASE}, """
CREATE DATABASE %(db)s;
""" % {'db': cls.DATABASE}]
for query in queries:
resp = _make_query(cls.client, query, database='default', local=False, server_name='impala')
resp = wait_for_query_to_finish(cls.client, resp, max=180.0)
content = json.loads(resp.content)
assert_true(content['status'] == 0, resp.content)
queries = ["""
CREATE TABLE tweets (row_num INTEGER, id_str STRING, text STRING) STORED AS PARQUET;
""", """
INSERT INTO TABLE tweets VALUES (1, "531091827395682000", "My dad looks younger than costa");
""", """
INSERT INTO TABLE tweets VALUES (2, "531091827781550000", "There is a thin line between your partner being vengeful and you reaping the consequences of your bad actions towards your partner.");
""", """
INSERT INTO TABLE tweets VALUES (3, "531091827768979000", "@Mustang_Sally83 and they need to get into you :))))");
""", """
INSERT INTO TABLE tweets VALUES (4, "531091827114668000", "@RachelZJohnson thank you rach!xxx");
""", """
INSERT INTO TABLE tweets VALUES (5, "531091827949309000", "i think @WWERollins was robbed of the IC title match this week on RAW also i wonder if he will get a rematch i hope so @WWE");
"""]
for query in queries:
resp = _make_query(cls.client, query, database=cls.DATABASE, local=False, server_name='impala')
resp = wait_for_query_to_finish(cls.client, resp, max=180.0)
content = json.loads(resp.content)
assert_true(content['status'] == 0, resp.content)
def test_thrift_over_http_config_with_proxy_endpoint():
resets = [
conf.SERVER_HOST.set_for_testing('impala_proxy'),
conf.SERVER_PORT.set_for_testing(36000),
conf.USE_THRIFT_HTTP.set_for_testing(True),
conf.PROXY_ENDPOINT.set_for_testing('/endpoint')
]
with patch('impala.dbms.get_hs2_http_port') as get_hs2_http_port:
get_hs2_http_port.return_value = 30000
try:
query_server = get_query_server_config(name='impala')
assert_equal(query_server['server_port'], 36000)
assert_equal(query_server['transport_mode'], 'http')
assert_equal(query_server['http_url'],
'http://impala_proxy:36000/endpoint')
finally:
for reset in resets:
reset()
def setUp(self):
self.finish = []
# We need a real Impala cluster currently
if not 'impala' in sys.argv and not os.environ.get('TEST_IMPALAD_HOST'):
raise SkipTest
if os.environ.get('TEST_IMPALAD_HOST'):
self.finish.append(SERVER_HOST.set_for_testing(os.environ.get('TEST_IMPALAD_HOST')))
self.client = make_logged_in_client()
self.user = User.objects.get(username='******')
add_to_group('test')
self.db = dbms.get(self.user, get_query_server_config(name='impala'))
hql = """
USE default;
DROP TABLE IF EXISTS %(db)s.tweets;
DROP DATABASE IF EXISTS %(db)s;
CREATE DATABASE %(db)s;
USE %(db)s;
""" % {'db': self.DATABASE}
resp = _make_query(self.client, hql, database='default', local=False, server_name='impala')
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
hql = """
CREATE TABLE tweets (row_num INTEGER, id_str STRING, text STRING) STORED AS PARQUET;
INSERT INTO TABLE tweets VALUES (1, "531091827395682000", "My dad looks younger than costa");
INSERT INTO TABLE tweets VALUES (2, "531091827781550000", "There is a thin line between your partner being vengeful and you reaping the consequences of your bad actions towards your partner.");
INSERT INTO TABLE tweets VALUES (3, "531091827768979000", "@Mustang_Sally83 and they need to get into you :))))");
INSERT INTO TABLE tweets VALUES (4, "531091827114668000", "@RachelZJohnson thank you rach!xxx");
INSERT INTO TABLE tweets VALUES (5, "531091827949309000", "i think @WWERollins was robbed of the IC title match this week on RAW also i wonder if he will get a rematch i hope so @WWE");
"""
resp = _make_query(self.client, hql, database=self.DATABASE, local=False, server_name='impala')
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
def tearDown(self):
for f in self.finish:
f()
def setUp(self):
user = User.objects.get(username='******')
self.db = dbms.get(user, get_query_server_config())
add_permission("test", "test", "write", "metastore")
def setUp(self):
user = User.objects.get(username='******')
self.db = dbms.get(user, get_query_server_config())
def setUp(self):
user = User.objects.get(username='******')
self.db = dbms.get(user, get_query_server_config())
add_permission("test", "test", "write", "metastore")
def setUp(self): user = User.objects.get(username='******') self.db = dbms.get(user, get_query_server_config())
