def drop_database(request):
source_type = request.POST.get('source_type',
request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
if request.POST.get('is_embeddable'):
design = SavedQuery.create_empty(
app_name=source_type
if source_type != 'hive' else 'beeswax',
owner=request.user,
data=hql_query('').dumps())
last_executed = json.loads(request.POST.get(
'start_time', '-1'))
cluster = json.loads(request.POST.get('cluster', '{}'))
namespace = json.loads(request.POST.get('namespace', '{}'))
sql = db.drop_databases(databases,
design,
generate_ddl_only=True)
job = make_notebook(name=_('Drop database %s') %
', '.join(databases)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
namespace=namespace,
compute=cluster,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
query_history = db.drop_databases(databases, design)
url = reverse(
'beeswax:watch_query_history',
kwargs={
'query_history_id': query_history.id
}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception as ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {
'databases': ','.join(databases),
'error': error_message
}
raise PopupException(error, title=_("DB Error"), detail=log)
else:
title = _("Do you really want to delete the database(s)?")
return render('confirm.mako', request, {
'url': request.path,
'title': title
})
def install(self, django_user):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name, ))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user,
name=self.name,
type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user,
name=self.name,
type=self.type,
desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' %
(self.name, ))
if beeswax.conf.USE_NEW_EDITOR.get():
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user,
name=self.name,
type=self._document_type(
self.type))
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query)
data = notebook.get_json()
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR)
doc2 = Document2.objects.create(owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=self._document_type(
self.type),
description=self.desc,
data=data)
# Share with default group
examples_dir.share(django_user,
Document2Permission.READ_PERM,
groups=[get_default_user_group()])
doc2.save()
LOG.info('Successfully installed sample query: %s' %
(self.name, ))
def install(self, django_user, interpreter=None):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
examples_dir = _get_example_directory(django_user)
document_type = self._document_type(self.type, interpreter)
notebook = import_saved_beeswax_query(query, interpreter=interpreter)
try:
# Could move PK from a uuid in queries.json at some point to handle name changes without duplicating.
# And move to a simpler json format at some point.
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)
if doc2.parent_directory != examples_dir: # Recover from Trash or if moved
doc2.parent_directory = examples_dir
data = json.loads(doc2.data)
data['uuid'] = doc2.uuid
doc2.data = json.dumps(data) # Refresh content
doc2.save()
except Document2.DoesNotExist:
# Create new example
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid, # Must the same as in the notebook data
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=document_type,
description=self.desc,
data=data
)
# TODO: FK to connector object
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % doc2)
def install(self, django_user, interpreter=None):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR
)
document_type = self._document_type(self.type, interpreter)
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query, interpreter=interpreter)
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid,
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=document_type,
description=self.desc,
data=data
)
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % (self.name,))
def install(self, django_user):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR
)
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=self._document_type(self.type))
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query)
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid,
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=self._document_type(self.type),
description=self.desc,
data=data
)
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % (self.name,))
def load_table(request, database, table):
db = dbms.get(request.user)
table = db.get_table(database, table)
response = {'status': -1, 'data': 'None'}
if request.method == "POST":
load_form = LoadDataForm(table, request.POST)
if load_form.is_valid():
on_success_url = reverse('metastore:describe_table',
kwargs={
'database': database,
'table': table.name
})
try:
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
query_history = db.load_data(database, table, load_form,
design)
url = reverse('beeswax:watch_query_history',
kwargs={'query_history_id': query_history.id
}) + '?on_success_url=' + on_success_url
response['status'] = 0
response['data'] = url
except Exception, e:
response['status'] = 1
response['data'] = _("Can't load the data: ") + str(e)
def drop_database(request):
db = dbms.get(request.user)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
if request.POST.get('is_embeddable'):
sql = db.drop_databases(databases, design, generate_ddl_only=True)
job = make_notebook(
name='Execute and watch',
editor_type='hive',
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True
)
return JsonResponse(job.execute(request))
else:
query_history = db.drop_databases(databases, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
skip_trash = request.POST.get('skip_trash') == 'on'
if request.POST.get('is_embeddable'):
sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True)
job = make_notebook(
name='Execute and watch',
editor_type='hive',
statement=sql.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True
)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_partition(request, database, table):
source_type = request.POST.get('source_type', 'hive')
db = _get_db(user=request.user, source_type=source_type)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
if request.GET.get("format", "html") == "json":
last_executed = json.loads(request.POST.get('start_time'), '-1')
sql = db.drop_partitions(database, table, partition_specs, design=None, generate_ddl_only=True)
job = make_notebook(
name=_('Drop partition %s') % ', '.join(partition_specs)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message}
raise PopupException(error, title=_("DB Error"), detail=log)
def drop_database(request):
db = dbms.get(request.user)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
query_history = db.drop_databases(databases, design)
url = reverse(
'beeswax:watch_query_history',
kwargs={
'query_history_id': query_history.id
}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {
'databases': ','.join(databases),
'error': error_message
}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_partition(request, database, table):
source_type = request.POST.get('source_type', 'hive')
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
if request.GET.get("format", "html") == "json":
last_executed = json.loads(request.POST.get('start_time'), '-1')
sql = db.drop_partitions(database, table, partition_specs, design=None, generate_ddl_only=True)
job = make_notebook(
name=_('Drop partition %s') % ', '.join(partition_specs)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message}
raise PopupException(error, title=_("DB Error"), detail=log)
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
skip_trash = request.POST.get('skip_trash') == 'on'
if request.POST.get('is_embeddable'):
sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True)
job = make_notebook(
name='Execute and watch',
editor_type='hive',
statement=sql.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True
)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_table(request, database):
source_type = request.POST.get('source_type',
request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [
db.get_table(database, table) for table in tables
]
skip_trash = request.POST.get('skip_trash') == 'on'
cluster = json.loads(request.POST.get('cluster', '{}'))
namespace = request.POST.get('namespace')
if request.POST.get('is_embeddable'):
last_executed = json.loads(request.POST.get('start_time'),
'-1')
sql = db.drop_tables(database,
tables_objects,
design=None,
skip_trash=skip_trash,
generate_ddl_only=True)
job = make_notebook(
name=_('Drop table %s') %
', '.join([table.name for table in tables_objects])[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=database,
namespace=namespace,
compute=cluster,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
query_history = db.drop_tables(database,
tables_objects,
design,
skip_trash=skip_trash)
url = reverse('beeswax:watch_query_history',
kwargs={'query_history_id': query_history.id
}) + '?on_success_url=' + reverse(
'metastore:show_tables',
kwargs={'database': database})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {
'tables': ','.join(tables),
'error': error_message
}
raise PopupException(error, title=_("DB Error"), detail=log)
def drop_database(request):
db = dbms.get(request.user)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
if request.POST.get('is_embeddable'):
sql = db.drop_databases(databases, design, generate_ddl_only=True)
job = make_notebook(
name='Execute and watch',
editor_type='hive',
statement=sql.strip(),
status='ready',
database=None,
on_success_url='assist.db.refresh',
is_task=True
)
return JsonResponse(job.execute(request))
else:
query_history = db.drop_databases(databases, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def create_saved_query(app_name, owner):
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = SavedQuery(owner=owner, type=query_type)
design.name = SavedQuery.DEFAULT_NEW_DESIGN_NAME
design.desc = ''
design.data = ''
design.is_auto = False
design.save()
return design
def create_saved_query(app_name, owner):
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = SavedQuery(owner=owner, type=query_type)
design.name = SavedQuery.DEFAULT_NEW_DESIGN_NAME
design.desc = ''
design.data = hql_query('show $tables', database='db1').dumps()
design.is_auto = False
design.save()
return design
def load_table(request, database, table):
db = dbms.get(request.user)
table = db.get_table(database, table)
response = {'status': -1, 'data': 'None'}
if request.method == "POST":
load_form = LoadDataForm(table, request.POST)
if load_form.is_valid():
on_success_url = reverse('metastore:describe_table',
kwargs={
'database': database,
'table': table.name
})
generate_ddl_only = request.POST.get('is_embeddable',
'false') == 'true'
try:
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
form_data = {
'path':
load_form.cleaned_data['path'],
'overwrite':
load_form.cleaned_data['overwrite'],
'partition_columns':
[(column_name, load_form.cleaned_data[key]) for key,
column_name in load_form.partition_columns.iteritems()],
}
query_history = db.load_data(
database,
table.name,
form_data,
design,
generate_ddl_only=generate_ddl_only)
if generate_ddl_only:
job = make_notebook(name='Execute and watch',
editor_type='hive',
statement=query_history.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True)
response = job.execute(request)
else:
url = reverse('beeswax:watch_query_history',
kwargs={
'query_history_id': query_history.id
}) + '?on_success_url=' + on_success_url
response['status'] = 0
response['data'] = url
response['query_history_id'] = query_history.id
except QueryError, ex:
response['status'] = 1
response['data'] = _("Can't load the data: ") + ex.message
except Exception, e:
response['status'] = 1
response['data'] = _("Can't load the data: ") + str(e)
def install(self, django_user):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name, ))
try:
# Don't overwrite
model = SavedQuery.objects.get(owner=django_user,
name=self.name,
type=self.type)
msg = _('Sample design %(name)s already exists.') % {
'name': self.name
}
LOG.error(msg)
raise InstallException(msg)
except SavedQuery.DoesNotExist:
model = SavedQuery(owner=django_user, name=self.name)
model.type = self.type
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
model.data = simplejson.dumps(self.data)
model.desc = self.desc
model.save()
LOG.info('Successfully installed sample design: %s' %
(self.name, ))
def create_saved_query(app_name, owner):
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = SavedQuery(owner=owner, type=query_type)
design.name = 'create_saved_query'
design.desc = ''
design.data = hql_query('show $tables', database='db1').dumps()
design.is_auto = False
design.save()
Document.objects.link(design, owner=design.owner, extra=design.type, name=design.name, description=design.desc)
return design
def create_saved_query(app_name, owner):
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = SavedQuery(owner=owner, type=query_type)
design.name = SavedQuery.DEFAULT_NEW_DESIGN_NAME
design.desc = ''
design.data = hql_query('show $tables', database='db1').dumps()
design.is_auto = False
design.save()
return design
def create_saved_query(app_name, owner):
query_type = SavedQuery.TYPES_MAPPING[app_name]
design = SavedQuery(owner=owner, type=query_type)
design.name = 'create_saved_query'
design.desc = ''
design.data = hql_query('show $tables', database='db1').dumps()
design.is_auto = False
design.save()
Document.objects.link(design, owner=design.owner, extra=design.type, name=design.name, description=design.desc)
return design
def drop_database(request):
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
if request.POST.get('is_embeddable'):
design = SavedQuery.create_empty(app_name=source_type if source_type != 'hive' else 'beeswax', owner=request.user, data=hql_query('').dumps())
last_executed = json.loads(request.POST.get('start_time'), '-1')
cluster = json.loads(request.POST.get('cluster', '{}'))
namespace = request.POST.get('namespace')
sql = db.drop_databases(databases, design, generate_ddl_only=True)
job = make_notebook(
name=_('Drop database %s') % ', '.join(databases)[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=None,
namespace=namespace,
compute=cluster,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_databases(databases, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message}
raise PopupException(error, title=_("DB Error"), detail=log)
def safe_get_design(request, design_type, design_id=None):
"""
Return a new design, if design_id is None,
Return the design with the given id and type. If the design is not found,
display a notification and return a new design.
"""
design = None
if design_id is not None:
design = authorized_get_design(request, design_id)
if design is None:
design = SavedQuery(owner=request.user, type=design_type)
return design
def load_table(request, database, table):
response = {'status': -1, 'data': 'None'}
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
table = db.get_table(database, table)
if request.method == "POST":
load_form = LoadDataForm(table, request.POST)
if load_form.is_valid():
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
generate_ddl_only = request.POST.get('is_embeddable', 'false') == 'true'
try:
design = SavedQuery.create_empty(app_name=source_type if source_type != 'hive' else 'beeswax', owner=request.user, data=hql_query('').dumps())
form_data = {
'path': load_form.cleaned_data['path'],
'overwrite': load_form.cleaned_data['overwrite'],
'partition_columns': [(column_name, load_form.cleaned_data[key]) for key, column_name in load_form.partition_columns.iteritems()],
}
query_history = db.load_data(database, table.name, form_data, design, generate_ddl_only=generate_ddl_only)
if generate_ddl_only:
last_executed = json.loads(request.POST.get('start_time'), '-1')
job = make_notebook(
name=_('Load data in %s.%s') % (database, table.name),
editor_type=source_type,
statement=query_history.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
response = job.execute(request)
else:
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + on_success_url
response['status'] = 0
response['data'] = url
response['query_history_id'] = query_history.id
except QueryError, ex:
response['status'] = 1
response['data'] = _("Can't load the data: ") + ex.message
except Exception, e:
response['status'] = 1
response['data'] = _("Can't load the data: ") + str(e)
def drop_partition(request, database, table):
db = dbms.get(request.user)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_database(request):
db = dbms.get(request.user)
if request.method == 'POST':
databases = request.POST.getlist('database_selection')
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_databases(databases, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_partition(request, database, table):
db = dbms.get(request.user)
if request.method == 'POST':
partition_specs = request.POST.getlist('partition_selection')
partition_specs = [spec for spec in partition_specs]
try:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \
reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == 'POST':
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user)
query_history = db.drop_tables(database, tables_objects, design)
url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('metastore:show_tables')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == 'POST':
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user)
query_history = db.drop_tables(database, tables_objects, design)
url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('catalog:show_tables')
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("Beeswax Error"), detail=log)
def load_table(request, database, table):
db = dbms.get(request.user)
table = db.get_table(database, table)
response = {'status': -1, 'data': 'None'}
if request.method == "POST":
load_form = LoadDataForm(table, request.POST)
if load_form.is_valid():
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
try:
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.load_data(database, table, load_form, design)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + on_success_url
response['status'] = 0
response['data'] = url
except Exception, e:
response['status'] = 1
response['data'] = _("Can't load the data: ") + str(e)
def load_table(request, database, table):
db = dbms.get(request.user)
table = db.get_table(database, table)
response = {"status": -1, "data": "None"}
if request.method == "POST":
load_form = LoadDataForm(table, request.POST)
if load_form.is_valid():
on_success_url = reverse("metastore:describe_table", kwargs={"database": database, "table": table.name})
try:
design = SavedQuery.create_empty(app_name="beeswax", owner=request.user)
query_history = db.load_data(database, table, load_form, design)
url = reverse("beeswax:watch_query", args=[query_history.id]) + "?on_success_url=" + on_success_url
response["status"] = 0
response["data"] = url
except Exception, e:
response["status"] = 1
response["data"] = _("Can't load the data: ") + str(e)
def drop_table(request, database):
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [db.get_table(database, table) for table in tables]
skip_trash = request.POST.get('skip_trash') == 'on'
cluster = json.loads(request.POST.get('cluster', '{}'))
namespace = request.POST.get('namespace')
if request.POST.get('is_embeddable'):
last_executed = json.loads(request.POST.get('start_time'), '-1')
sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True)
job = make_notebook(
name=_('Drop table %s') % ', '.join([table.name for table in tables_objects])[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=database,
namespace=namespace,
compute=cluster,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps())
query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash)
url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database})
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message}
raise PopupException(error, title=_("DB Error"), detail=log)
def drop_partition(request, database, table):
db = dbms.get(request.user)
if request.method == "POST":
partition_specs = request.POST.getlist("partition_selection")
partition_specs = [spec for spec in partition_specs]
try:
design = SavedQuery.create_empty(app_name="beeswax", owner=request.user, data=hql_query("").dumps())
query_history = db.drop_partitions(database, table, partition_specs, design)
url = (
reverse("beeswax:watch_query_history", kwargs={"query_history_id": query_history.id})
+ "?on_success_url="
+ reverse("metastore:describe_partitions", kwargs={"database": database, "table": table})
)
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(partition)s. Error: %(error)s") % {
"partition": "\n".join(partition_specs),
"error": error_message,
}
raise PopupException(error, title=_("Hive Error"), detail=log)
def install(self, django_user):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
model = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
model = SavedQuery(owner=django_user, name=self.name)
model.type = self.type
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
model.data = json.dumps(self.data)
model.desc = self.desc
model.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
def drop_database(request):
db = dbms.get(request.user)
if request.method == "POST":
databases = request.POST.getlist("database_selection")
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name="beeswax", owner=request.user, data=hql_query("").dumps())
query_history = db.drop_databases(databases, design)
url = (
reverse("beeswax:watch_query_history", kwargs={"query_history_id": query_history.id})
+ "?on_success_url="
+ reverse("metastore:databases")
)
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(databases)s. Error: %(error)s") % {
"databases": ",".join(databases),
"error": error_message,
}
raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_table(request, database):
db = dbms.get(request.user)
if request.method == "POST":
tables = request.POST.getlist("table_selection")
tables_objects = [db.get_table(database, table) for table in tables]
try:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name="beeswax", owner=request.user)
query_history = db.drop_tables(database, tables_objects, design)
url = (
reverse("beeswax:watch_query", args=[query_history.id])
+ "?on_success_url="
+ reverse("metastore:show_tables")
)
return redirect(url)
except Exception, ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {
"tables": ",".join(tables),
"error": error_message,
}
raise PopupException(error, title=_("Beeswax Error"), detail=log)
def load_table(request, database, table):
response = {'status': -1, 'data': 'None'}
source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
table = db.get_table(database, table)
if request.method == "POST":
load_form = LoadDataForm(table, request.POST)
if load_form.is_valid():
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': table.name})
generate_ddl_only = request.POST.get('is_embeddable', 'false') == 'true'
try:
design = SavedQuery.create_empty(
app_name=source_type if source_type != 'hive' else 'beeswax',
owner=request.user,
data=hql_query('').dumps()
)
form_data = {
'path': load_form.cleaned_data['path'],
'overwrite': load_form.cleaned_data['overwrite'],
'partition_columns': [(column_name, load_form.cleaned_data[key]) for key, column_name in load_form.partition_columns.items()],
}
query_history = db.load_data(database, table.name, form_data, design, generate_ddl_only=generate_ddl_only)
if generate_ddl_only:
last_executed = json.loads(request.POST.get('start_time'), '-1')
job = make_notebook(
name=_('Load data in %s.%s') % (database, table.name),
editor_type=source_type,
statement=query_history.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed
)
response = job.execute(request)
else:
url = reverse(
'beeswax:watch_query_history',
kwargs={'query_history_id': query_history.id}
) + '?on_success_url=' + on_success_url
response['status'] = 0
response['data'] = url
response['query_history_id'] = query_history.id
except QueryError as ex:
response['status'] = 1
response['data'] = _("Can't load the data: ") + ex.message
except Exception as e:
response['status'] = 1
response['data'] = _("Can't load the data: ") + str(e)
else:
load_form = LoadDataForm(table)
if response['status'] == -1:
popup = render('popups/load_data.mako', request, {
'table': table,
'load_form': load_form,
'source_type': source_type,
'database': database,
'app_name': 'beeswax'
}, force_template=True).content
response['data'] = popup
return JsonResponse(response)