def fetch_result_size(self, notebook, snippet):
resp = {'rows': None, 'size': None, 'message': ''}
if snippet.get('status') != 'available':
raise QueryError(_('Result status is not available'))
if has_connectors():
# TODO: Add dialect to snippet and update fetchResultSize() in notebook.ko
interpreter = get_interpreter(connector_type=snippet['type'])
snippet_dialect = interpreter['dialect']
else:
snippet_dialect = snippet['type']
if snippet_dialect not in ('hive', 'impala'):
raise OperationNotSupported(
_('Cannot fetch result metadata for snippet type: %s') %
snippet_dialect)
if snippet_dialect == 'hive':
resp['rows'], resp['size'], resp[
'message'] = self._get_hive_result_size(notebook, snippet)
else:
resp['rows'], resp['size'], resp[
'message'] = self._get_impala_result_size(notebook, snippet)
return resp
def install_examples(request):
response = {'status': -1, 'message': '', 'errorMessage': ''}
try:
connector = Connector.objects.get(id=request.POST.get('connector'))
if connector:
dialect = connector.dialect
db_name = request.POST.get('db_name', 'default')
interpreter = get_interpreter(connector_type=connector.to_dict()['type'], user=request.user)
successes, errors = beeswax_install_examples.Command().handle(
dialect=dialect, db_name=db_name, user=request.user, interpreter=interpreter, request=request
)
response['message'] = ' '.join(successes)
response['errorMessage'] = ' '.join(errors)
response['status'] = len(errors)
else:
Command().handle(user=request.user, dialect=request.POST.get('dialect', 'hive'))
response['status'] = 0
response['message'] = _('Examples refreshed')
except Exception as e:
msg = 'Error during Editor samples installation'
LOG.exception(msg)
response['errorMessage'] = msg + ': ' + str(e)
return JsonResponse(response)
def install_examples(request):
response = {'status': -1, 'message': ''}
if request.method == 'POST':
try:
connector = Connector.objects.get(id=request.POST.get('connector'))
if connector:
app_name = 'beeswax' if connector.dialect == 'hive' else 'impala'
db_name = request.POST.get('db_name', 'default')
interpreter = get_interpreter(
connector_type=connector.to_dict()['type'],
user=request.user)
beeswax_install_examples.Command().handle(
app_name=app_name,
db_name=db_name,
user=request.user,
interpreter=interpreter)
else:
Command().handle(user=request.user)
response['status'] = 0
except Exception as err:
LOG.exception(err)
response['message'] = str(err)
else:
response['message'] = _('A POST request is required.')
return JsonResponse(response)
def get_context_computes(request, interface):
'''
Some clusters like Snowball can have multiple computes for a certain languages (Hive, Impala...).
'''
response = {}
computes = []
clusters = list(get_clusters(request.user).values())
if get_cluster_config(request.user)['has_computes']: # TODO: only based on interface selected?
interpreter = get_interpreter(connector_type=interface, user=request.user)
if interpreter['dialect'] == 'impala':
# dw_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters']
dw_clusters = [
{'crn': 'c1', 'clusterName': 'c1', 'status': 'created', 'options': {'server_host': 'c1.gethue.com', 'server_port': 10000}},
{'crn': 'c2', 'clusterName': 'c2', 'status': 'created', 'options': {'server_host': 'c2.gethue.com', 'server_port': 10000}},
]
computes.extend([{
'id': cluster.get('crn'),
'name': cluster.get('clusterName'),
'status': cluster.get('status'),
'namespace': cluster.get('namespaceCrn', cluster.get('crn')),
'type': interpreter['dialect'],
'options': cluster['options'],
} for cluster in dw_clusters]
)
else:
# Currently broken if not sent
computes.extend([{
'id': cluster['id'],
'name': cluster['name'],
'namespace': cluster['id'],
'interface': interface,
'type': cluster['type'],
'options': {}
} for cluster in clusters if cluster.get('type') == 'direct'
])
response[interface] = computes
response['status'] = 0
return JsonResponse(response)
def create_table_from_local_file(self, source, destination, start_time=-1):
if '.' in destination['name']:
database, table_name = destination['name'].split('.', 1)
else:
database = 'default'
table_name = destination['name']
final_table_name = table_name
source_type = source['sourceType']
editor_type = destination['sourceType']
columns = destination['columns']
dialect = get_interpreter(source_type, self.user)['dialect']
if dialect in ('hive', 'mysql'):
if dialect == 'mysql':
for col in columns:
if col['type'] == 'string':
col['type'] = 'VARCHAR(255)'
sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
%(columns)s);
''' % {
'database':
database,
'table_name':
table_name,
'columns':
',\n'.join([' `%(name)s` %(type)s' % col for col in columns]),
}
elif dialect == 'phoenix':
for col in columns:
if col['type'] == 'string':
col['type'] = 'CHAR(255)'
sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
%(columns)s
CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s));
''' % {
'database':
database,
'table_name':
table_name,
'columns':
',\n'.join([' %(name)s %(type)s' % col for col in columns]),
'primary_keys':
', '.join(destination.get('primaryKeys'))
}
elif dialect == 'impala':
sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s_tmp (
%(columns)s);
''' % {
'database':
database,
'table_name':
table_name,
'columns':
',\n'.join([' `%(name)s` string' % col for col in columns]),
} # Impala does not implicitly cast between string and numeric or Boolean types.
path = urllib_unquote(source['path'])
if path: # data insertion
with open(path, 'r') as local_file:
reader = csv.reader(local_file)
_csv_rows = []
for count, row in enumerate(reader):
if (source['format']['hasHeader']
and count == 0) or not row:
continue
if dialect == 'impala': # for the boolean col updating csv_val to (1,0)
row = self.nomalize_booleans(row, columns)
_csv_rows.append(tuple(row))
if _csv_rows:
csv_rows = str(_csv_rows)[1:-1]
if dialect in ('hive', 'mysql'):
sql += '''\nINSERT INTO %(database)s.%(table_name)s VALUES %(csv_rows)s;
''' % {
'database': database,
'table_name': table_name,
'csv_rows': csv_rows
}
elif dialect == 'phoenix':
for csv_row in _csv_rows:
_sql = ', '.join([ "'{0}'".format(col_val) if columns[count]['type'] in ('CHAR(255)', 'timestamp') \
else '{0}'.format(col_val) for count, col_val in enumerate(csv_row)])
sql += '''\nUPSERT INTO %(database)s.%(table_name)s VALUES (%(csv_row)s);
''' % {
'database': database,
'table_name': table_name,
'csv_row': _sql
}
elif dialect == 'impala':
# casting from string to boolean is not allowed in impala so string -> int -> bool
sql_ = ',\n'.join([
' CAST ( `%(name)s` AS %(type)s ) `%(name)s`' % col if col['type'] != 'boolean' \
else ' CAST ( CAST ( `%(name)s` AS TINYINT ) AS boolean ) `%(name)s`' % col for col in columns
])
sql += '''\nINSERT INTO %(database)s.%(table_name)s_tmp VALUES %(csv_rows)s;\n\nCREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s
AS SELECT\n%(sql_)s\nFROM %(database)s.%(table_name)s_tmp;\n\nDROP TABLE IF EXISTS %(database)s.%(table_name)s_tmp;''' % {
'database': database,
'table_name': table_name,
'csv_rows': csv_rows,
'sql_': sql_
}
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \
'?source_type=' + source_type
return make_notebook(name=_('Creating table %(database)s.%(table)s') %
{
'database': database,
'table': final_table_name
},
editor_type=editor_type,
statement=sql.strip(),
status='ready',
database=database,
on_success_url=on_success_url,
last_executed=start_time,
is_task=True)
def make_notebook(name='Browse',
description='',
editor_type='hive',
statement='',
status='ready',
files=None,
functions=None,
settings=None,
is_saved=False,
database='default',
snippet_properties=None,
batch_submit=False,
on_success_url=None,
skip_historify=False,
is_task=False,
last_executed=-1,
is_notebook=False,
pub_sub_url=None,
result_properties={},
namespace=None,
compute=None,
is_presentation_mode=False):
'''
skip_historify: do not add the task to the query history. e.g. SQL Dashboard
is_task / isManaged: true when being a managed by Hue operation (include_managed=True in document),
e.g. exporting query result, dropping some tables
'''
from notebook.connectors.hiveserver2 import HS2Api
if has_connectors():
interpreter = get_interpreter(connector_type=editor_type)
editor_connector = editor_type
editor_type = interpreter['dialect']
else:
editor_connector = editor_type
editor = Notebook()
if snippet_properties is None:
snippet_properties = {}
if editor_type == 'hive':
sessions_properties = HS2Api.get_properties(editor_type)
if files is not None:
_update_property_value(sessions_properties, 'files', files)
if functions is not None:
_update_property_value(sessions_properties, 'functions', functions)
if settings is not None:
_update_property_value(sessions_properties, 'settings', settings)
elif editor_type == 'impala':
sessions_properties = HS2Api.get_properties(editor_type)
if settings is not None:
_update_property_value(sessions_properties, 'files', files)
elif editor_type == 'java':
sessions_properties = [] # Java options
else:
sessions_properties = []
data = {
'name':
name,
'uuid':
str(uuid.uuid4()),
'description':
description,
'sessions': [{
'type': editor_connector,
'properties': sessions_properties,
'id': None
}],
'selectedSnippet':
editor_connector, # TODO: might need update in notebook.ko.js
'type':
'notebook' if is_notebook else 'query-%s' % editor_type,
'showHistory':
True,
'isSaved':
is_saved,
'onSuccessUrl':
urllib_quote(on_success_url.encode('utf-8'), safe=SAFE_CHARACTERS_URI)
if on_success_url else None,
'pubSubUrl':
pub_sub_url,
'skipHistorify':
skip_historify,
'isPresentationModeDefault':
is_presentation_mode,
'isManaged':
is_task,
'snippets': [{
'status': status,
'id': str(uuid.uuid4()),
'statement_raw': statement,
'statement': statement,
'type': editor_connector,
'wasBatchExecuted': batch_submit,
'lastExecuted': last_executed,
'properties': {
'files': [] if files is None else files,
'functions': [] if functions is None else functions,
'settings': [] if settings is None else settings
},
'name': name,
'database': database,
'namespace': namespace if namespace else {},
'compute': compute if compute else {},
'result': {
'handle': {}
},
'variables': []
}] if not is_notebook else []
}
if has_connectors(): # To improve
data['dialect'] = interpreter['dialect']
data['type'] = '%s-%s' % (editor_type, editor_connector
) # e.g. 'flink-' + editor_connector
if snippet_properties:
data['snippets'][0]['properties'].update(snippet_properties)
if result_properties:
data['snippets'][0]['result'].update(result_properties)
editor.data = json.dumps(data)
return editor
def create_table_from_local_file(self, source, destination, start_time=-1):
if '.' in destination['name']:
database, table_name = destination['name'].split('.', 1)
else:
database = 'default'
table_name = destination['name']
final_table_name = table_name
source_type = source['sourceType']
editor_type = destination['sourceType']
columns = destination['columns']
dialect = get_interpreter(source_type, self.user)['dialect']
if dialect in ('hive', 'mysql'):
if dialect == 'mysql':
for col in columns:
if col['type'] == 'string':
col['type'] = 'VARCHAR(255)'
sql = '''CREATE TABLE IF NOT EXISTS %(database)s.%(table_name)s (
%(columns)s);
''' % {
'database':
database,
'table_name':
table_name,
'columns':
',\n'.join([' `%(name)s` %(type)s' % col for col in columns]),
}
path = urllib_unquote(source['path'])
if path: # data insertion
with open(BASE_DIR + path, 'r') as local_file:
reader = csv.reader(local_file)
list_of_tuples = list(map(tuple, reader))
if source['format']['hasHeader']:
list_of_tuples = list_of_tuples[1:]
csv_rows = str(list_of_tuples)[1:-1]
if dialect in ('hive', 'mysql'):
sql += '''\nINSERT INTO %(database)s.%(table_name)s VALUES %(csv_rows)s;
''' % {
'database': database,
'table_name': table_name,
'csv_rows': csv_rows
}
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \
'?source_type=' + source_type
return make_notebook(name=_('Creating table %(database)s.%(table)s') %
{
'database': database,
'table': final_table_name
},
editor_type=editor_type,
statement=sql.strip(),
status='ready',
database=database,
on_success_url=on_success_url,
last_executed=start_time,
is_task=True)