def execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
table = db.execute_statement(
snippet['statement']) # TODO: execute statement stub in Rdbms
data = table.rows()
has_result_set = data is not None
return {
'sync': True,
'has_result_set': has_result_set,
'modified_row_count': 0,
'result': {
'has_more':
False,
'data':
data if has_result_set else [],
'meta': [{
'name': col,
'type': '',
'comment': ''
} for col in table.columns] if has_result_set else [],
'type':
'table'
}
}
def explain_query(request):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
try:
form = get_query_form(request)
if form.is_valid():
query = SQLdesign(form, query_type=query_type)
query_server = dbms.get_query_server_config(request.POST.get('server'))
db = dbms.get(request.user, query_server)
try:
db.use(form.cleaned_data['database'])
datatable = db.explain(query)
results = db.client.create_result(datatable)
response['status'] = 0
response['results'] = results_to_dict(results)
except Exception, e:
response['status'] = -1
response['message'] = str(e)
else:
def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
query_server = dbms.get_query_server_config(server='mysql')
db = MySQLClient(query_server, self.user)
assist = Assist(db)
response = {'error': 0}
try:
if database is None:
response['databases'] = assist.get_databases()
elif table is None:
response['tables'] = assist.get_tables(database)
else:
columns = assist.get_columns(database, table)
response['columns'] = [col for col in columns]
response['extended_columns'] = [{
'name': col,
'type': '',
'comment': ''
} for col in columns
]
except Exception, e:
LOG.warn('Autocomplete data fetching error: %s' % e)
response['code'] = -1
response['error'] = str(e)
def explain(self, notebook, snippet):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
db.use(snippet['database'])
result = db.explain(snippet['statement'])
rows = list(result.rows())
cols = result.cols()
# Prettify output
explanation = ""
cols_pretty = [(col + ": ") for col in cols]
col_width = max(len(col) for col in cols_pretty)
for index, col in enumerate(cols_pretty):
lines = []
for row in rows:
lines += str(row[index]).split("\n")
explanation += col.ljust(col_width) + lines[0] + "\n"
for line in lines[1:]:
explanation += (" " * col_width) + line + "\n"
return {
'status': 0,
'explanation': explanation,
'statement': snippet['statement'],
}
def autocomplete(self,
snippet,
database=None,
table=None,
column=None,
nested=None):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
assist = Assist(db)
response = {'status': -1}
if database is None:
response['databases'] = assist.get_databases()
elif table is None:
tables_meta = []
for t in assist.get_tables(database):
tables_meta.append({'name': t, 'type': 'Table', 'comment': ''})
response['tables_meta'] = tables_meta
else:
columns = assist.get_columns(database, table)
response['columns'] = [col['name'] for col in columns]
response['extended_columns'] = columns
response['status'] = 0
return response
def autocomplete(self,
snippet,
database=None,
table=None,
column=None,
nested=None):
query_server = dbms.get_query_server_config(server='mysql')
db = MySQLClient(query_server, self.user)
assist = Assist(db)
response = {'error': 0}
try:
if database is None:
response['databases'] = assist.get_databases()
elif table is None:
response['tables'] = assist.get_tables(database)
else:
columns = assist.get_columns(database, table)
response['columns'] = [col for col in columns]
response['extended_columns'] = [{
'name': col,
'type': '',
'comment': ''
} for col in columns]
except Exception, e:
LOG.warn('Autocomplete data fetching error: %s' % e)
response['code'] = -1
response['error'] = str(e)
def execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server='mysql')
db = MySQLClient(query_server, self.user)
table = db.execute_statement(snippet['statement'])
data = table.rows()
has_result_set = data is not None
return {
'sync': True,
'has_result_set': has_result_set,
'result': {
'has_more':
False,
'data':
data if has_result_set else [],
'meta': [{
'name': col,
'type': '',
'comment': ''
} for col in table.columns] if has_result_set else [],
'type':
'table'
}
}
def explain_query(request):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
try:
form = get_query_form(request)
if form.is_valid():
query = SQLdesign(form, query_type=query_type)
query_server = dbms.get_query_server_config(
request.POST.get('server'))
db = dbms.get(request.user, query_server)
try:
db.use(form.cleaned_data['database'])
datatable = db.explain(query)
results = db.client.create_result(datatable)
response['status'] = 0
response['results'] = results_to_dict(results)
except Exception, e:
response['status'] = -1
response['message'] = str(e)
else:
def get_sample_data(self,
mode=None,
database=None,
table=None,
column=None):
if self.db:
db = self.db
else:
query_server = rdbms.get_query_server_config(
server=self.db_conf_name)
db = rdbms.get(self.user, query_server=query_server)
if mode == 'configRdbms' or self.db_conf_name != 'jdbc':
assist = Assist(db)
else:
assist = JdbcAssist(db)
response = {'status': -1}
sample_data = assist.get_sample_data(database, table, column)
if sample_data:
response['status'] = 0
response['headers'] = sample_data.columns
response['rows'] = list(sample_data.rows())
else:
response['message'] = _('Failed to get sample data.')
return response
def explain_query(request):
response = {"status": -1, "message": ""}
if request.method != "POST":
response["message"] = _("A POST request is required.")
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
try:
form = get_query_form(request)
if form.is_valid():
query = SQLdesign(form, query_type=query_type)
query_server = dbms.get_query_server_config(request.POST.get("server"))
db = dbms.get(request.user, query_server)
try:
db.use(form.cleaned_data["database"])
datatable = db.explain(query)
results = db.client.create_result(datatable)
response["status"] = 0
response["results"] = results_to_dict(results)
except Exception, e:
response["status"] = -1
response["message"] = str(e)
else:
def execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
db.use(snippet['database'])
table = db.execute_statement(snippet['statement']) # TODO: execute statement stub in Rdbms
data = list(table.rows())
has_result_set = data is not None
return {
'sync': True,
'has_result_set': has_result_set,
'modified_row_count': 0,
'result': {
'has_more': False,
'data': data if has_result_set else [],
'meta': [{
'name': col,
'type': '',
'comment': ''
} for col in table.columns] if has_result_set else [],
'type': 'table'
}
}
def execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
db.use(
snippet['database']
) # TODO: only do the use on the first statement in a multi query
table = db.execute_statement(
snippet['statement']) # TODO: execute statement stub in Rdbms
data = list(table.rows())
has_result_set = data is not None
print table.columns
print table.columns_description
return {
'sync': True,
'has_result_set': has_result_set,
'modified_row_count': 0,
'result': {
'has_more':
False,
'data':
data if has_result_set else [],
'meta': [{
'name': col['name'],
'type': col.get('type', ''),
'comment': ''
} for col in table.columns_description]
if has_result_set else [],
'type':
'table'
}
}
def execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server="mysql")
db = dbms.get(self.user, query_server)
table = db.execute_statement()
print table
return {"result": table}
def _execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
db.use(snippet['database']) # TODO: only do the use on the first statement in a multi query
table = db.execute_statement(snippet['statement']) # TODO: execute statement stub in Rdbms
return table
def execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server='mysql')
db = dbms.get(self.user, query_server)
table = db.execute_statement()
print table
return {'result': table}
def setup_class(cls):
if not ENABLE_SQOOP.get():
raise SkipTest
if not rdbms.get_query_server_config(server='mysql'):
raise SkipTest
cls.client = make_logged_in_client()
cls.user = User.objects.get(username='******')
cls.user = rewrite_user(cls.user)
cls.indexer = RdbmsIndexer(cls.user, db_conf_name='mysql')
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
stream = request.fs.open(file_format["path"])
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": file_format['path']
},
"format": file_format['format']
})
elif file_format['inputFormat'] == 'table':
sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
db = dbms.get(request.user)
table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
format_ = {
"sample": sample['rows'][:4],
"columns": [
Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict()
for col in table_metadata.cols
]
}
elif file_format['inputFormat'] == 'query': # Only support open query history
# TODO get schema from explain query, which is not possible
notebook = Notebook(document=Document2.objects.get(id=file_format['query'])).get_data()
snippet = notebook['snippets'][0]
sample = get_api(request, snippet).fetch_result(notebook, snippet, 4, start_over=True)
format_ = {
"sample": sample['rows'][:4],
"sample_cols": sample.meta,
"columns": [
Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
for col in sample.meta
]
}
elif file_format['inputFormat'] == 'rdbms':
query_server = rdbms.get_query_server_config(server=file_format['rdbmsType'])
db = rdbms.get(request.user, query_server=query_server)
sample = RdbmsIndexer(request.user, file_format['rdbmsType']).get_sample_data(mode=file_format['rdbmsMode'], database=file_format['rdbmsDatabaseName'], table=file_format['rdbmsTableName'])
table_metadata = db.get_columns(file_format['rdbmsDatabaseName'], file_format['rdbmsTableName'], names_only=False)
format_ = {
"sample": list(sample['rows'])[:4],
"columns": [
Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
for col in table_metadata
]
}
return JsonResponse(format_)
def databases(request, server):
query_server = dbms.get_query_server_config(server)
if not query_server:
raise Http404
db = dbms.get(request.user, query_server)
response = {"status": 0, "databases": db.get_databases()}
return HttpResponse(json.dumps(response), mimetype="application/json")
def databases(request, server):
query_server = dbms.get_query_server_config(server)
if not query_server:
raise Http404
db = dbms.get(request.user, query_server)
response = {'status': 0, 'databases': db.get_databases()}
return HttpResponse(json.dumps(response), mimetype="application/json")
def databases(request, server):
query_server = dbms.get_query_server_config(server)
if not query_server:
raise Http404
db = dbms.get(request.user, query_server)
response = {'status': 0, 'databases': db.get_databases()}
return JsonResponse(response)
def tables(request, server, database):
query_server = dbms.get_query_server_config(server)
if not query_server:
raise Http404
db = dbms.get(request.user, query_server)
db.use(database)
response = {'tables': db.get_tables(database)}
return HttpResponse(json.dumps(response), mimetype="application/json")
def columns(request, server, database, table):
query_server = dbms.get_query_server_config(server)
if not query_server:
raise Http404
db = dbms.get(request.user, query_server)
db.use(database)
response = {"columns": db.get_columns(database, table)}
return HttpResponse(json.dumps(response), mimetype="application/json")
def setup_class(cls):
if not ENABLE_SQOOP.get():
raise SkipTest
if not rdbms.get_query_server_config(server='mysql'):
raise SkipTest
cls.client = make_logged_in_client()
cls.user = User.objects.get(username='******')
cls.user = rewrite_user(cls.user)
request = Bag()
request.user = cls.user
request.POST = {'source': '{"rdbmsMode":"configRdbms", "rdbmsType": "mysql", "inputFormat": "rdbms"}'}
cls.indexer = _get_api(request)
def tables(request, server, database):
query_server = dbms.get_query_server_config(server)
if not query_server:
raise Http404
db = dbms.get(request.user, query_server)
db.use(database)
response = {'tables': db.get_tables(database)}
return JsonResponse(response)
def get_db_component(request):
format_ = {'data': [], 'status': 1, 'message': ''}
db = None
try:
source = json.loads(request.POST.get('source', '{}'))
user = User.objects.get(username=request.user)
name = None
if source['rdbmsMode'] == 'configRdbms':
if source['rdbmsType'] != 'jdbc':
query_server = rdbms.get_query_server_config(server=source['rdbmsType'])
db = rdbms.get(user, query_server=query_server)
else:
interpreters = get_ordered_interpreters(request.user)
options = {}
key = [key for key in interpreters if key['name'] == source['rdbmsJdbcDriverName']]
if key:
options = key[0]['options']
db = Jdbc(driver_name=options['driver'], url=options['url'], username=options['user'], password=options['password'])
else:
name = source['rdbmsType']
if name != 'jdbc':
query_server = {
'server_name': name,
'server_host': source['rdbmsHostname'],
'server_port': int(source['rdbmsPort']),
'username': source['rdbmsUsername'],
'password': source['rdbmsPassword'],
'options': {},
'alias': name
}
db = rdbms.get(user, query_server=query_server)
else:
db = Jdbc(driver_name=source['rdbmsJdbcDriver'], url=source['rdbmsHostname'], username=source['rdbmsUsername'], password=source['rdbmsPassword'])
if source['rdbmsType'] != 'jdbc':
assist = Assist(db)
else:
assist = JdbcAssist(db)
if not source['rdbmsDatabaseName']:
data = assist.get_databases()
elif source['rdbmsDatabaseName']:
data = assist.get_tables(source['rdbmsDatabaseName'])
format_['data'] = [{'name': element, 'value': element} for element in data]
format_['status'] = 0
except Exception, e:
message = _('Error accessing the database %s: %s') % (name, e)
LOG.warn(message)
format['message'] = message
def tables(request, server, database):
query_server = dbms.get_query_server_config(server)
if not query_server:
raise Http404
db = dbms.get(request.user, query_server)
db.use(database)
response = {
'tables': db.get_tables(database)
}
return JsonResponse(response)
def databases(request, server):
query_server = dbms.get_query_server_config(server)
if not query_server:
raise Http404
db = dbms.get(request.user, query_server)
response = {
'status': 0,
'databases': db.get_databases()
}
return JsonResponse(response)
def get_query_server_config_from_request(request):
check_params = {"server_name": request.GET.get("server_type")}
if "host" in request.GET:
check_params["server_host"] = request.GET.get("host")
if "port" in request.GET:
check_params["server_port"] = request.GET.get("port")
if "username" in request.GET:
check_params["username"] = request.GET.get("username")
for alias in librdbms_conf.DATABASES:
config = dbms.get_query_server_config(alias)
if all([check_params[param] == config[param] for param in check_params]):
return config
return None
def _get_api(request):
file_format = json.loads(
request.POST.get('source', request.POST.get('fileFormat', '{}')))
options = None
query_server = None
if file_format['rdbmsMode'] == 'customRdbms':
type = 'custom'
if file_format['rdbmsType'] == 'jdbc':
name = file_format[
'rdbmsHostname'] # We make sure it's unique as name is the cache key
interface = file_format['rdbmsType']
options = {
'driver': file_format['rdbmsJdbcDriver'],
'url': file_format['rdbmsHostname'],
'user': file_format['rdbmsUsername'],
'password': file_format['rdbmsPassword']
}
else:
interface = 'rdbms'
query_server = {
'server_name': file_format['rdbmsType'],
'server_host': file_format['rdbmsHostname'],
'server_port': int(file_format['rdbmsPort'] or '3306'),
'username': file_format['rdbmsUsername'],
'password': file_format['rdbmsPassword'],
'options': {},
'alias': file_format['rdbmsType']
}
name = 'rdbms:%(server_name)s://%(server_host)s:%(server_port)s' % query_server # We make sure it's unique as name is the cache key
else:
if file_format['rdbmsType'] == 'jdbc':
type = file_format['rdbmsJdbcDriverName'] and file_format[
'rdbmsJdbcDriverName'].lower()
else:
type = file_format['rdbmsType']
query_server = rdbms.get_query_server_config(
server=file_format['rdbmsType'])
name = type
interface = file_format['inputFormat']
return get_api(
request, {
'type': type,
'interface': interface,
'options': options,
'query_server': query_server,
'name': name
})
def get_query_server_config_from_request(request):
check_params = {'server_name': request.GET.get('server_type')}
if 'host' in request.GET:
check_params['server_host'] = request.GET.get('host')
if 'port' in request.GET:
check_params['server_port'] = request.GET.get('port')
if 'username' in request.GET:
check_params['username'] = request.GET.get('username')
for alias in librdbms_conf.DATABASES:
config = dbms.get_query_server_config(alias)
if all(
[check_params[param] == config[param] for param in check_params]):
return config
return None
def get_query_server_config_from_request(request):
check_params = {
'server_name': request.GET.get('server_type')
}
if 'host' in request.GET:
check_params['server_host'] = request.GET.get('host')
if 'port' in request.GET:
check_params['server_port'] = request.GET.get('port')
if 'username' in request.GET:
check_params['username'] = request.GET.get('username')
for alias in librdbms_conf.DATABASES:
config = dbms.get_query_server_config(alias)
if all([check_params[param] == config[param] for param in check_params]):
return config
return None
def get_sample_data(self, snippet, database=None, table=None, column=None):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
assist = Assist(db)
response = {'status': -1}
sample_data = assist.get_sample_data(database, table, column)
if sample_data:
response['status'] = 0
response['headers'] = sample_data.columns
response['rows'] = list(sample_data.rows())
else:
response['message'] = _('Failed to get sample data.')
return response
def get_sample_data(self, snippet, database=None, table=None, column=None):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
assist = Assist(db)
response = {'status': -1}
sample_data = assist.get_sample_data(database, table, column)
if sample_data:
response['status'] = 0
response['headers'] = sample_data.columns
response['rows'] = list(sample_data.rows())
else:
response['message'] = _('Failed to get sample data.')
return response
def get_sample_data(self, snippet, database=None, table=None):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
assist = Assist(db)
response = {"status": -1}
sample_data = assist.get_sample_data(database, table)
if sample_data:
response["status"] = 0
response["headers"] = sample_data.columns
response["rows"] = list(sample_data.rows())
else:
response["message"] = _("Failed to get sample data.")
return response
def _get_db(request):
source = json.loads(
request.POST.get('source', request.POST.get('fileFormat', '{}')))
user = User.objects.get(username=request.user)
name = None
if source['rdbmsMode'] == 'configRdbms':
if source['rdbmsType'] != 'jdbc':
query_server = rdbms.get_query_server_config(
server=source['rdbmsType'])
db = rdbms.get(user, query_server=query_server)
else:
interpreters = get_ordered_interpreters(request.user)
options = {}
key = [
key for key in interpreters
if key['name'] == source['rdbmsJdbcDriverName']
]
if key:
options = key[0]['options']
db = Jdbc(driver_name=options['driver'],
url=options['url'],
username=options['user'],
password=options['password'])
else:
name = source['rdbmsType']
if name != 'jdbc':
query_server = {
'server_name': name,
'server_host': source['rdbmsHostname'],
'server_port': int(source['rdbmsPort']),
'username': source['rdbmsUsername'],
'password': source['rdbmsPassword'],
'options': {},
'alias': name
}
db = rdbms.get(user, query_server=query_server)
else:
db = Jdbc(driver_name=source['rdbmsJdbcDriver'],
url=source['rdbmsHostname'],
username=source['rdbmsUsername'],
password=source['rdbmsPassword'])
return db
def execute_query(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
try:
form = get_query_form(request, design_id)
if form.is_valid():
design = save_design(request, SaveForm(), form, query_type, design)
query = SQLdesign(form, query_type=query_type)
query_server = dbms.get_query_server_config(
request.POST.get('server'))
db = dbms.get(request.user, query_server)
query_history = db.execute_query(query, design)
query_history.last_state = beeswax_models.QueryHistory.STATE.expired.value
query_history.save()
try:
db.use(form.cleaned_data['database'])
datatable = db.execute_and_wait(query)
results = db.client.create_result(datatable)
response['status'] = 0
response['results'] = results_to_dict(results)
response['design'] = design.id
except Exception as e:
response['status'] = -1
response['message'] = str(e)
else:
response['message'] = _('There was an error with your query.')
response['errors'] = form.errors
except RuntimeError as e:
response['message'] = str(e)
return JsonResponse(response, encoder=ResultEncoder)
def get_query_form(request, design_id=None):
servers = conf.get_server_choices()
# Get database choices
query_server = dbms.get_query_server_config(request.POST.get('server'))
if not query_server:
raise RuntimeError(_("Server specified doesn't exist."))
db = dbms.get(request.user, query_server)
databases = [(database, database) for database in db.get_databases()]
if not databases:
raise RuntimeError(_("No databases are available. Permissions could be missing."))
form = SQLForm(request.POST)
form.fields['server'].choices = servers # Could not do it in the form
form.fields['database'].choices = databases # Could not do it in the form
return form
def get_query_form(request, design_id=None):
servers = conf.get_server_choices()
# Get database choices
query_server = dbms.get_query_server_config(request.POST.get('server'))
if not query_server:
raise RuntimeError(_("Server specified doesn't exist."))
db = dbms.get(request.user, query_server)
databases = [(database, database) for database in db.get_databases()]
if not databases:
raise RuntimeError(_("No databases are available. Permissions could be missing."))
form = SQLForm(request.POST)
form.fields['server'].choices = servers # Could not do it in the form
form.fields['database'].choices = databases # Could not do it in the form
return form
def execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
table = db.execute_statement(snippet["statement"]) # TODO: execute statement stub in Rdbms
data = table.rows()
has_result_set = data is not None
return {
"sync": True,
"has_result_set": has_result_set,
"modified_row_count": 0,
"result": {
"has_more": False,
"data": data if has_result_set else [],
"meta": [{"name": col, "type": "", "comment": ""} for col in table.columns] if has_result_set else [],
"type": "table",
},
}
def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
assist = Assist(db)
response = {'status': -1}
if database is None:
response['databases'] = assist.get_databases()
elif table is None:
tables_meta = []
for t in assist.get_tables(database):
tables_meta.append({'name': t, 'type': 'Table', 'comment': ''})
response['tables_meta'] = tables_meta
else:
columns = assist.get_columns(database, table)
response['columns'] = [col['name'] for col in columns]
response['extended_columns'] = columns
response['status'] = 0
return response
def autocomplete(self, snippet, database=None, table=None, column=None, nested=None):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
assist = Assist(db)
response = {"status": -1}
if database is None:
response["databases"] = assist.get_databases()
elif table is None:
tables_meta = []
for t in assist.get_tables(database):
tables_meta.append({"name": t, "type": "Table", "comment": ""})
response["tables_meta"] = tables_meta
else:
columns = assist.get_columns(database, table)
response["columns"] = [col["name"] for col in columns]
response["extended_columns"] = columns
response["status"] = 0
return response
def execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server='mysql')
db = MySQLClient(query_server, self.user)
table = db.execute_statement(snippet['statement'])
data = table.rows()
has_result_set = data is not None
return {
'sync': True,
'has_result_set': has_result_set,
'result': {
'has_more': False,
'data': data if has_result_set else [],
'meta': [{
'name': col,
'type': '',
'comment': ''
} for col in table.columns] if has_result_set else [],
'type': 'table'
}
}
def get_db_component(request):
format_ = {'data': [], 'status': 1, 'message': ''}
name = None
try:
source = json.loads(request.POST.get('source', '{}'))
user = User.objects.get(username=request.user)
if source['rdbmsMode'] == 'configRdbms':
query_server = rdbms.get_query_server_config(
server=source['rdbmsType'])
else:
name = source['rdbmsType']
if name:
query_server = {
'server_name': name,
'server_host': source['rdbmsHostname'],
'server_port': int(source['rdbmsPort']),
'username': source['rdbmsUsername'],
'password': source['rdbmsPassword'],
'options': {},
'alias': name
}
db = rdbms.get(user, query_server=query_server)
assist = Assist(db)
if not source['rdbmsDatabaseName']:
data = assist.get_databases()
else:
data = assist.get_tables(source['rdbmsDatabaseName'])
format_['data'] = [{
'name': element,
'value': element
} for element in data]
format_['status'] = 0
except Exception, e:
message = 'Error accessing the database %s' % name
LOG.warn(message, e)
format['message'] = _(message)
def execute_query(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
try:
form = get_query_form(request, design_id)
if form.is_valid():
design = save_design(request, SaveForm(), form, query_type, design)
query = SQLdesign(form, query_type=query_type)
query_server = dbms.get_query_server_config(request.POST.get('server'))
db = dbms.get(request.user, query_server)
query_history = db.execute_query(query, design)
query_history.last_state = beeswax_models.QueryHistory.STATE.expired.index
query_history.save()
try:
db.use(form.cleaned_data['database'])
datatable = db.execute_and_wait(query)
results = db.client.create_result(datatable)
response['status'] = 0
response['results'] = results_to_dict(results)
response['design'] = design.id
except Exception, e:
response['status'] = -1
response['message'] = str(e)
else:
def _get_query_server(self):
if self.query_server:
return self.query_server
else:
return dbms.get_query_server_config(server=self.interpreter)
try:
sample = db.fetch_result(notebook, snippet, 4, start_over=True)['rows'][:4]
except Exception, e:
LOG.warn('Skipping sample data as query handle might be expired: %s' % e)
sample = [[], [], [], [], []]
columns = db.autocomplete(snippet=snippet, database='', table='')
columns = [
Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
for col in columns['extended_columns']
]
format_ = {
"sample": sample,
"columns": columns,
}
elif file_format['inputFormat'] == 'rdbms':
query_server = rdbms.get_query_server_config(server=file_format['rdbmsType'])
db = rdbms.get(request.user, query_server=query_server)
sample = RdbmsIndexer(request.user, file_format['rdbmsType']).get_sample_data(mode=file_format['rdbmsMode'], database=file_format['rdbmsDatabaseName'], table=file_format['rdbmsTableName'])
table_metadata = db.get_columns(file_format['rdbmsDatabaseName'], file_format['rdbmsTableName'], names_only=False)
format_ = {
"sample": list(sample['rows'])[:4],
"columns": [
Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
for col in table_metadata
]
}
elif file_format['inputFormat'] == 'stream':
# Note: mocked here, should come from SFDC or Kafka API or sampling job
if file_format['streamSelection'] == 'kafka':
data = """%(kafkaFieldNames)s
class RdbmsApi(Api):
def _execute(self, notebook, snippet):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
db.use(
snippet['database']
) # TODO: only do the use on the first statement in a multi query
table = db.execute_statement(
snippet['statement']) # TODO: execute statement stub in Rdbms
return table
@query_error_handler
def execute(self, notebook, snippet):
table = self._execute(notebook, snippet)
data = list(table.rows())
has_result_set = data is not None
return {
'sync': True,
'has_result_set': has_result_set,
'modified_row_count': 0,
'result': {
'has_more':
False,
'data':
data if has_result_set else [],
'meta': [{
'name': col['name'] if type(col) is dict else col,
'type': col.get('type', '') if type(col) is dict else '',
'comment': ''
} for col in table.columns_description]
if has_result_set else [],
'type':
'table'
}
}
@query_error_handler
def check_status(self, notebook, snippet):
return {'status': 'expired'}
@query_error_handler
def fetch_result(self, notebook, snippet, rows, start_over):
return {'has_more': False, 'data': [], 'meta': [], 'type': 'table'}
@query_error_handler
def fetch_result_metadata(self):
pass
@query_error_handler
def cancel(self, notebook, snippet):
return {'status': 0}
@query_error_handler
def get_log(self, notebook, snippet, startFrom=None, size=None):
return 'No logs'
@query_error_handler
def download(self, notebook, snippet, format, user_agent=None):
file_name = _get_snippet_name(notebook)
results = self._execute(notebook, snippet)
db = FixedResult(results)
return data_export.download(None,
format,
db,
id=snippet['id'],
file_name=file_name,
user_agent=user_agent)
@query_error_handler
def close_statement(self, snippet):
return {'status': -1}
@query_error_handler
def autocomplete(self,
snippet,
database=None,
table=None,
column=None,
nested=None):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
assist = Assist(db)
response = {'status': -1}
if database is None:
response['databases'] = assist.get_databases()
elif table is None:
tables_meta = []
for t in assist.get_tables(database):
tables_meta.append({'name': t, 'type': 'Table', 'comment': ''})
response['tables_meta'] = tables_meta
elif column is None:
columns = assist.get_columns(database, table)
response['columns'] = [col['name'] for col in columns]
response['extended_columns'] = columns
else:
columns = assist.get_columns(database, table)
response['name'] = next(
(col['name'] for col in columns if column == col['name']), '')
response['type'] = next(
(col['type'] for col in columns if column == col['name']), '')
response['status'] = 0
return response
@query_error_handler
def get_sample_data(self,
snippet,
database=None,
table=None,
column=None,
async=False):
query_server = dbms.get_query_server_config(server=self.interpreter)
db = dbms.get(self.user, query_server)
assist = Assist(db)
response = {'status': -1}
sample_data = assist.get_sample_data(database, table, column)
if sample_data:
response['status'] = 0
response['headers'] = sample_data.columns
response['full_headers'] = sample_data.columns_description
response['rows'] = list(sample_data.rows())
else:
response['message'] = _('Failed to get sample data.')
return response
