def upload_history(request):
response = {'status': -1}
if request.user.is_superuser:
api = OptimizerApi(request.user)
histories = []
upload_stats = {}
if request.POST.get('sourcePlatform'):
n = min(request.POST.get('n', OPTIMIZER.QUERY_HISTORY_UPLOAD_LIMIT.get()))
source_platform = request.POST.get('sourcePlatform', 'hive')
histories = [(source_platform, Document2.objects.get_history(doc_type='query-%s' % source_platform, user=request.user)[:n])]
elif OPTIMIZER.QUERY_HISTORY_UPLOAD_LIMIT.get() > 0:
histories = [
(source_platform, Document2.objects.filter(type='query-%s' % source_platform, is_history=True, is_managed=False, is_trashed=False).order_by('-last_modified')[:OPTIMIZER.QUERY_HISTORY_UPLOAD_LIMIT.get()])
for source_platform in ['hive', 'impala']
]
for source_platform, history in histories:
queries = _convert_queries([Notebook(document=doc).get_data() for doc in history])
upload_stats[source_platform] = api.upload(data=queries, data_type='queries', source_platform=source_platform)
response['upload_history'] = upload_stats
response['status'] = 0
else:
response['message'] = _('Query history upload requires Admin privileges or feature is disabled.')
return JsonResponse(response)
def top_tables(request):
response = {'status': -1}
database = request.POST.get('database', 'default')
limit = request.POST.get('len', 1000)
api = OptimizerApi()
data = api.top_tables(database_name=database, page_size=limit)
tables = [{
'eid': table['eid'],
'database': _get_table_name(table['name'])['database'],
'name': _get_table_name(table['name'])['table'],
'popularity': table['workloadPercent'],
'column_count': table['columnCount'],
'patternCount': table['patternCount'],
'total': table['total'],
'is_fact': table['type'] != 'Dimension'
} for table in data['results']
]
if NAVIGATOR.APPLY_SENTRY_PERMISSIONS.get():
checker = PrivilegeChecker(user=request.user)
action = 'SELECT'
for table in tables:
paths = _get_table_name(table['name'])
table.update({u'db': paths['database'], u'table': paths['table'], u'column': None, u'server': u'server1'})
tables = list(checker.filter_objects(tables, action)) #, getkey=getkey)
response['top_tables'] = tables
response['status'] = 0
return JsonResponse(response)
def top_tables(request):
response = {'status': -1}
database = request.POST.get('database', 'default')
limit = request.POST.get('len', 1000)
api = OptimizerApi(user=request.user)
data = api.top_tables(database_name=database, page_size=limit)
tables = [{
'eid': table['eid'],
'database': _get_table_name(table['name'])['database'],
'name': _get_table_name(table['name'])['table'],
'popularity': table['workloadPercent'],
'column_count': table['columnCount'],
'patternCount': table['patternCount'],
'total': table['total'],
'is_fact': table['type'] != 'Dimension'
} for table in data['results']
]
response['top_tables'] = tables
response['status'] = 0
return JsonResponse(response)
def upload_query(request):
response = {'status': -1}
source_platform = request.POST.get('sourcePlatform', 'default')
if OPTIMIZER.AUTO_UPLOAD_QUERIES.get() and source_platform in ('hive',
'impala'):
query_id = request.POST.get('query_id')
doc = Document2.objects.document(request.user, doc_id=query_id)
query_data = Notebook(document=doc).get_data()
queries = _convert_queries([query_data])
source_platform = query_data['snippets'][0]['type']
api = OptimizerApi(request.user)
response['query_upload'] = api.upload(data=queries,
data_type='queries',
source_platform=source_platform)
else:
response['query_upload'] = _('Skipped')
response['status'] = 0
return JsonResponse(response)
def top_tables(request):
response = {'status': -1}
api = OptimizerApi()
response['top_tables'] = api.top_tables()
response['status'] = 0
return JsonResponse(response)
def top_tables(request):
response = {'status': -1}
api = OptimizerApi()
response['top_tables'] = api.top_tables()
response['status'] = 0
return JsonResponse(response)
def table_details(request):
response = {'status': -1}
table_name = request.POST.get('tableName')
api = OptimizerApi()
response['table_details'] = api.table_details(table_name=table_name)
response['status'] = 0
return JsonResponse(response)
def table_details(request):
response = {'status': -1}
table_name = request.POST.get('tableName')
api = OptimizerApi()
response['table_details'] = api.table_details(table_name=table_name)
response['status'] = 0
return JsonResponse(response)
def upload_status(request):
response = {'status': -1}
workload_id = request.POST.get('workloadId')
api = OptimizerApi()
response['upload_status'] = api.upload_status(workload_id=workload_id)
response['status'] = 0
return JsonResponse(response)
def top_databases(request):
response = {'status': -1}
api = OptimizerApi()
data = api.top_databases()
if data['status'] == 'success':
response['status'] = 0
response['values'] = data['results']
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def popular_joins(request):
response = {'status': -1}
database_name = request.POST.get('databaseName')
api = OptimizerApi()
data = api.popular_joins(database_name=database_name)
if data['status'] == 'success':
response['status'] = 0
response['values'] = data['results']
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def get_tenant(request):
response = {'status': -1}
email = request.POST.get('email')
api = OptimizerApi()
data = api.get_tenant(email=email)
if data['status'] == 'success':
response['status'] = 0
response['data'] = data['tenant']
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def top_aggs(request):
response = {'status': -1}
db_tables = json.loads(request.POST.get('dbTables'), '[]')
api = OptimizerApi(request.user)
data = api.top_aggs(db_tables=db_tables)
if data:
response['status'] = 0
response['values'] = data['results']
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def get_tenant(request):
response = {'status': -1}
cluster_id = request.POST.get('cluster_id')
api = OptimizerApi(request.user)
data = api.get_tenant(cluster_id=cluster_id)
if data:
response['status'] = 0
response['data'] = data['tenant']
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def top_columns(request):
response = {'status': -1}
db_tables = json.loads(request.POST.get('dbTables'), '[]')
api = OptimizerApi()
data = api.top_columns(db_tables=db_tables)
if data['status'] == 'success':
response['status'] = 0
response['values'] = data
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def query_risk(request):
response = {'status': -1}
query = json.loads(request.POST.get('query'))
api = OptimizerApi()
data = api.query_risk(query=query)
if data['status'] == 'success':
response['status'] = 0
response['query_risk'] = data
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def upload_history(request):
response = {'status': -1}
query_type = 'hive'
queries = [
(doc.uuid, 1000, Notebook(document=doc).get_data()['snippets'][0]['statement'])
for doc in Document2.objects.get_history(doc_type='query-%s' % query_type, user=request.user)[:25]
]
api = OptimizerApi()
response['upload_history'] = api.upload(queries=queries, source_platform=query_type)
response['status'] = 0
return JsonResponse(response)
def table_details(request):
response = {'status': -1}
table_name = request.POST.get('tableName')
api = OptimizerApi()
data = api.table_details(table_name=table_name)
if data['status'] == 'success':
response['status'] = 0
response['details'] = data['details']
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def upload_history(request):
response = {"status": -1}
query_type = "hive"
queries = [
(doc.uuid, 1000, Notebook(document=doc).get_data()["snippets"][0]["statement"])
for doc in Document2.objects.get_history(doc_type="query-%s" % query_type, user=request.user)[:25]
]
api = OptimizerApi()
response["upload_history"] = api.upload(queries=queries, source_platform=query_type)
response["status"] = 0
return JsonResponse(response)
def statement_compatibility(self, notebook, snippet, source_platform, target_platform):
response = self._get_current_statement(notebook, snippet)
query = response['statement']
api = OptimizerApi(self.user)
return api.query_compatibility(source_platform, target_platform, query)
def statement_risk(self, notebook, snippet):
response = self._get_current_statement(notebook, snippet)
query = response['statement']
api = OptimizerApi(self.user)
return api.query_risk(query=query, source_platform=snippet['type'], db_name=snippet.get('database') or 'default')
def statement_similarity(self, notebook, snippet, source_platform):
response = self._get_current_statement(notebook, snippet)
query = response['statement']
api = OptimizerApi(self.user)
return api.similar_queries(source_platform, query)
def table_details(request):
response = {'status': -1}
table_name = request.POST.get('tableName')
api = OptimizerApi()
data = api.table_details(table_name=table_name)
if data['status'] == 'success':
response['status'] = 0
response['details'] = data['details']
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def top_filters(request):
response = {'status': -1}
db_tables = json.loads(request.POST.get('dbTables'), '[]')
column_name = request.POST.get('columnName') # Unsused
api = OptimizerApi()
data = api.top_filters(db_tables=db_tables)
if data['status'] == 'success':
response['status'] = 0
response['values'] = data['results']
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def table_details(request):
response = {"status": -1}
table_name = request.POST.get("tableName")
api = OptimizerApi()
data = api.table_details(table_name=table_name)
if data["status"] == "success":
response["status"] = 0
response["details"] = data["details"]
else:
response["message"] = "Optimizer: %s" % data["details"]
return JsonResponse(response)
def upload_history(request):
response = {'status': -1}
query_type = 'hive'
queries = [
(doc.uuid, 1000, Notebook(document=doc).get_data()['snippets'][0]['statement'])
for doc in Document2.objects.get_history(doc_type='query-%s' % query_type, user=request.user)[:25]
]
api = OptimizerApi()
response['upload_history'] = api.upload(queries=queries, source_platform=query_type)
response['status'] = 0
return JsonResponse(response)
def query_risk(request):
response = {'status': -1}
query = json.loads(request.POST.get('query'))
source_platform = request.POST.get('sourcePlatform')
api = OptimizerApi()
data = api.query_risk(query=query, source_platform=source_platform)
if data:
response['status'] = 0
response['query_risk'] = data
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def simiar_queries(request):
response = {'status': -1}
source_platform = request.POST.get('sourcePlatform')
query = request.POST.get('query')
api = OptimizerApi()
data = api.similar_queries(source_platform=source_platform, query=query)
if data['status'] == 'success':
response['status'] = 0
response['similar_queries'] = json.loads(data['details']['similarQueries'])
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def similar_queries(request):
response = {"status": -1}
source_platform = request.POST.get("sourcePlatform")
query = request.POST.get("query")
api = OptimizerApi()
data = api.similar_queries(source_platform=source_platform, query=query)
if data["status"] == "success":
response["status"] = 0
response["similar_queries"] = json.loads(data["details"]["similarQueries"])
else:
response["message"] = "Optimizer: %s" % data["details"]
return JsonResponse(response)
def table_details(request):
response = {'status': -1}
database_name = request.POST.get('databaseName')
table_name = request.POST.get('tableName')
api = OptimizerApi(request.user)
data = api.table_details(database_name=database_name, table_name=table_name)
if data:
response['status'] = 0
response['details'] = data
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def similar_queries(request):
response = {'status': -1}
source_platform = request.POST.get('sourcePlatform')
query = json.loads(request.POST.get('query'))
api = OptimizerApi()
data = api.similar_queries(source_platform=source_platform, query=query)
if data['status'] == 'success':
response['status'] = 0
response['similar_queries'] = data
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def top_tables(request):
response = {"status": -1}
database = request.POST.get("database", "default")
len = request.POST.get("len", 1000)
if OPTIMIZER.MOCKING.get():
from beeswax.server import dbms
from beeswax.server.dbms import get_query_server_config
db = dbms.get(request.user)
tables = [
{
"name": table,
"popularity": random.randint(1, 100),
"column_count": random.randint(1, 100),
"is_fact": bool(random.getrandbits(1)),
}
for table in db.get_tables(database=database)
][:len]
else:
"""
Get back:
# u'details': [{u'columnCount': 28, u'name': u'date_dim', u'patternCount': 136, u'workloadPercent': 89, u'total': 92, u'type': u'Dimension', u'eid': u'19'},
"""
api = OptimizerApi()
data = api.top_tables()
tables = [
{
"eid": table["eid"],
"name": table["name"],
"popularity": table["workloadPercent"],
"column_count": table["columnCount"],
"patternCount": table["patternCount"],
"total": table["total"],
"is_fact": table["type"] != "Dimension",
}
for table in data["details"]
]
response["top_tables"] = tables
response["status"] = 0
return JsonResponse(response)
def query_compatibility(request):
response = {'status': -1}
source_platform = request.POST.get('sourcePlatform')
target_platform = request.POST.get('targetPlatform')
query = request.POST.get('query')
api = OptimizerApi(request.user)
data = api.query_compatibility(source_platform=source_platform, target_platform=target_platform, query=query)
if data:
response['status'] = 0
response['query_compatibility'] = data
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def statement_similarity(self, notebook, snippet, source_platform):
db = self._get_db(snippet, cluster=snippet.get('selectedCompute'))
response = self._get_current_statement(db, snippet)
query = response['statement']
api = OptimizerApi(self.user)
return api.similar_queries(source_platform, query)
def setup_class(cls):
cls.client = make_logged_in_client(username='******', is_superuser=False)
cls.user = User.objects.get(username='******')
cls.user = rewrite_user(cls.user)
add_to_group('test')
grant_access("test", "test", "metadata")
grant_access("test", "test", "optimizer")
cls.api = OptimizerApi()
def query_compatibility(request):
response = {'status': -1}
source_platform = request.POST.get('sourcePlatform')
target_platform = request.POST.get('targetPlatform')
query = request.POST.get('query')
api = OptimizerApi()
data = api.query_compatibility(source_platform=source_platform, target_platform=target_platform, query=query)
if data['status'] == 'success':
response['status'] = 0
response['query_compatibility'] = json.loads(data['details'])
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def query_compatibility(request):
response = {'status': -1}
source_platform = request.POST.get('sourcePlatform')
target_platform = request.POST.get('targetPlatform')
query = request.POST.get('query')
api = OptimizerApi()
data = api.query_compatibility(source_platform=source_platform, target_platform=target_platform, query=query)
if data['status'] == 'success':
response['status'] = 0
response['query_compatibility'] = json.loads(data['details'])
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def query_compatibility(request):
response = {"status": -1}
source_platform = request.POST.get("sourcePlatform")
target_platform = request.POST.get("targetPlatform")
query = request.POST.get("query")
api = OptimizerApi()
data = api.query_compatibility(source_platform=source_platform, target_platform=target_platform, query=query)
if data["status"] == "success":
response["status"] = 0
response["query_compatibility"] = json.loads(data["details"])
else:
response["message"] = "Optimizer: %s" % data["details"]
return JsonResponse(response)
def popular_values(request):
response = {'status': -1}
table_name = request.POST.get('tableName')
column_name = request.POST.get('columnName')
api = OptimizerApi()
data = api.popular_filter_values(table_name=table_name, column_name=column_name)
if data['status'] == 'success':
if 'status' in data['details']:
response['values'] = [] # Bug in Opt API
else:
response['values'] = data['details']
response['status'] = 0
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
def popular_values(request):
response = {'status': -1}
database_name = request.POST.get('databaseName')
table_name = request.POST.get('tableName')
column_name = request.POST.get('columnName') # Unsused
api = OptimizerApi()
data = api.popular_filter_values(database_name=database_name,
table_name=table_name,
column_name=column_name)
if data['status'] == 'success':
response['status'] = 0
response['values'] = data['results']
else:
response['message'] = 'Optimizer: %s' % data
return JsonResponse(response)
def setup_class(cls):
if not is_live_cluster() or not is_optimizer_enabled():
raise SkipTest
cls.client = make_logged_in_client(username='******', is_superuser=False)
cls.user = User.objects.get(username='******')
cls.user = rewrite_user(cls.user)
add_to_group('test')
grant_access("test", "test", "metadata")
grant_access("test", "test", "optimizer")
cls.api = OptimizerApi()
def top_tables(request):
response = {'status': -1}
database = request.POST.get('database', 'default')
len = request.POST.get('len', 1000)
if OPTIMIZER.MOCKING.get():
from beeswax.server import dbms
from beeswax.server.dbms import get_query_server_config
db = dbms.get(request.user)
tables = [
{'name': table, 'popularity': random.randint(1, 100) , 'column_count': random.randint(1, 100), 'is_fact': bool(random.getrandbits(1))}
for table in db.get_tables(database=database)
][:len]
else:
"""
Get back:
# u'details': [{u'columnCount': 28, u'name': u'date_dim', u'patternCount': 136, u'workloadPercent': 89, u'total': 92, u'type': u'Dimension', u'eid': u'19'},
"""
api = OptimizerApi()
data = api.top_tables()
tables = [{
'eid': table['eid'],
'name': table['name'],
'popularity': table['workloadPercent'],
'column_count': table['columnCount'],
'patternCount': table['patternCount'],
'total': table['total'],
'is_fact': table['type'] != 'Dimension'
} for table in data['details']
]
response['top_tables'] = tables
response['status'] = 0
return JsonResponse(response)
def popular_values(request):
response = {'status': -1}
table_name = request.POST.get('tableName')
column_name = request.POST.get('columnName')
if OPTIMIZER.MOCKING.get():
if column_name:
values = [
{
"values": [
"1",
"(6,0)"
],
"columnName": "d_dow",
"tableName": "date_dim"
}
]
else:
values = [
{
"values": [
"('2001q1','2001q2','2001q3')",
"'2001q1'"
],
"columnName": "d_quarter_name",
"tableName": "date_dim"
},
{
"values": [
"1",
"2",
"4"
],
"columnName": "d_qoy",
"tableName": "date_dim"
},
{
"values": [
"Subquery"
],
"columnName": "d_week_seq",
"tableName": "date_dim"
},
{
"values": [
"(cast('1998-08-14' as date) + interval '30' day)",
"(cast ('1998-03-08' as date) + interval '30' day)",
"d1.d_date + 5",
"cast('1998-08-14' as date)",
"cast('1999-04-26' as date)",
"'2002-4-01'",
"(cast('2000-02-02' as date) + interval '90' day)",
"(cast('2002-4-01' as date) + interval '60' day)",
"(cast('2002-01-18' as date) + 60 + interval '60' day)",
"('1999-04-17','1999-10-04','1999-11-10')",
"(cast('1999-04-26' as date) + 30 + interval '30' day)",
"(cast('1999-06-03' as date) + interval '30' day)",
"cast('1998-01-06' as date)",
"(cast('2000-2-01' as date) + interval '60' day)",
"(cast('2002-04-01' as date) + interval '30' day)",
"( cast('2000-03-22' as date ) + interval '90' day )",
"cast('2001-08-21' as date)",
"(cast ('1998-03-08' as date) - interval '30' day)",
"'2000-03-22'",
"(cast('2001-08-21' as date) + interval '14' day)",
"( cast('1999-08-25' as date) + interval '30' day )",
"Subquery",
"'2000-3-01'",
"cast('2002-01-18' as date)",
"(cast ('2001-03-14' as date) - interval '30' day)",
"'2000-02-02'",
"cast('2002-04-01' as date)",
"'2002-03-09'",
"(cast('2000-3-01' as date) + interval '60' day)",
"cast('1999-06-03' as date)",
"cast('1999-08-25' as date)",
"(cast ('2001-03-14' as date) + interval '30' day)",
"'2000-2-01'",
"(cast('1998-01-06' as date) + interval '60' day)"
],
"columnName": "d_date",
"tableName": "date_dim"
},
{
"values": [
"1223",
"1200",
"1202",
"1214+11",
"(select distinct date_dim.d_month_seq+1 from date_dim where date_dim.d_year = 2001 and date_dim.d_moy = 5)",
"1181+11",
"1199",
"1191",
"(1206,1206+1,1206+2,1206+3,1206+4,1206+5,1206+6,1206+7,1206+8,1206+9,1206+10,1206+11)",
"1211 + 11",
"1199 + 11",
"1212",
"(select distinct date_dim.d_month_seq+3 from date_dim where date_dim.d_year = 2001 and date_dim.d_moy = 5)",
"1211",
"1214",
"Subquery",
"(1195,1195+1,1195+2,1195+3,1195+4,1195+5,1195+6,1195+7,1195+8,1195+9,1195+10,1195+11)",
"1200+11",
"1212 + 11",
"1223+11",
"1183 + 11",
"1183",
"1181",
"1191 + 11",
"1202 + 11"
],
"columnName": "d_month_seq",
"tableName": "date_dim"
},
{
"values": [
"11",
"4 + 3",
"12",
"3+2",
"2+3",
"1",
"3",
"2",
"5",
"4",
"6",
"8",
"10"
],
"columnName": "d_moy",
"tableName": "date_dim"
},
{
"values": [
"25",
"16",
"28",
"1",
"3",
"2"
],
"columnName": "d_dom",
"tableName": "date_dim"
},
{
"values": [
"(1998,1998+1)",
"2000 + 1",
"2000 + 2",
"(2000,2000+1,2000+2)",
"(1999,1999+1,1999+2)",
"2000-1",
"2001+1",
"1999 + 2",
"2000+1",
"2000+2",
"1999+1",
"(2002)",
"( 1999, 1999 + 1, 1999 + 2, 1999 + 3 )",
"1999-1",
"( 1998, 1998 + 1, 1998 + 2 )",
"1999",
"1998",
"(1998,1998+1,1998+2)",
"2002",
"2000",
"2001",
"2004"
],
"columnName": "d_year",
"tableName": "date_dim"
},
{
"values": [
"1",
"(6,0)"
],
"columnName": "d_dow",
"tableName": "date_dim"
}
]
else:
api = OptimizerApi()
data = api.popular_filter_values(table_name=table_name, column_name=column_name)
if data['status'] == 'success':
if 'status' in data['details']:
response['values'] = [] # Bug in Opt API
else:
response['values'] = data['details']
response['status'] = 0
else:
response['message'] = 'Optimizer: %s' % data['details']
return JsonResponse(response)
