def _save_results_ctas(request, query_history, target_table, result_meta): """ Handle saving results as a new table. Returns HTTP response. May raise BeeswaxException, IOError. """ query_server = query_history.get_query_server() # Query server requires DDL support # Case 1: The results are straight from an existing table if result_meta.in_tablename: hql = 'CREATE TABLE `%s` AS SELECT * FROM %s' % (target_table, result_meta.in_tablename) query_msg = make_beeswax_query(request, hql) # Display the CTAS running. Could take a long time. return execute_directly(request, query_msg, query_server, on_success_url=urlresolvers.reverse(show_tables)) # Case 2: The results are in some temporary location # 1. Create table cols = '' schema = result_meta.schema for i, field in enumerate(schema.fieldSchemas): if i != 0: cols += ',\n' cols += '`%s` %s' % (field.name, field.type) # The representation of the delimiter is messy. # It came from Java as a string, which might has been converted from an integer. # So it could be "1" (^A), or "10" (\n), or "," (a comma literally). delim = result_meta.delim if not delim.isdigit(): delim = str(ord(delim)) hql = ''' CREATE TABLE `%s` ( %s ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\%s' STORED AS TextFile ''' % (target_table, cols, delim.zfill(3)) query_msg = make_beeswax_query(request, hql) db_utils.execute_and_wait(request.user, query_msg, query_server) try: # 2. Move the results into the table's storage table_obj = db_utils.meta_client().get_table("default", target_table) table_loc = request.fs.urlsplit(table_obj.sd.location)[2] request.fs.rename_star(result_meta.table_dir, table_loc) LOG.debug("Moved results from %s to %s" % (result_meta.table_dir, table_loc)) messages.info(request, _('Saved query results as new table %(table)s') % {'table': target_table}) query_history.save_state(models.QueryHistory.STATE.expired) except Exception, ex: LOG.error('Error moving data into storage of table %s. Will drop table.' % (target_table,)) query_msg = make_beeswax_query(request, 'DROP TABLE `%s`' % (target_table,)) try: db_utils.execute_directly(request.user, query_msg, query_server) # Don't wait for results except Exception, double_trouble: LOG.exception('Failed to drop table "%s" as well: %s' % (target_table, double_trouble))
def describe_table(request, table): table_obj = db_utils.meta_client().get_table("default", table) sample_results = None is_view = table_obj.tableType == 'VIRTUAL_VIEW' # Don't show samples if it's a view (HUE-526). if not is_view: # Show the first few rows hql = "SELECT * FROM `%s` %s" % (table, _get_browse_limit_clause(table_obj)) query_msg = make_beeswax_query(request, hql) try: sample_results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0) except: # Gracefully degrade if we're unable to load the results. logging.exception("Failed to read table '%s'" % table) sample_results = None hdfs_link = location_to_url(request, table_obj.sd.location) load_form = beeswax.forms.LoadDataForm(table_obj) return render("describe_table.mako", request, dict( table=table_obj, table_name=table, top_rows=sample_results and list(parse_results(sample_results.data)) or None, hdfs_link=hdfs_link, load_form=load_form, is_view=is_view ))
def load(self, django_user): """ Load data into table. Raise InstallException on failure. """ LOAD_HQL = \ """ LOAD DATA local INPATH '%(filename)s' OVERWRITE INTO TABLE %(tablename)s """ LOG.info('Loading data into table "%s"' % (self.name, )) hql = LOAD_HQL % dict(tablename=self.name, filename=self._contents_file) query_msg = _make_query_msg(hql) try: results = db_utils.execute_and_wait(django_user, query_msg) if not results: msg = _( 'Error loading table %(table)s: Operation timeout.') % { 'table': self.name } LOG.error(msg) raise InstallException(msg) except BeeswaxException, ex: msg = _('Error loading table %(table)s: %(error)s') % { 'table': self.name, 'error': ex } LOG.error(msg) raise InstallException(msg)
def create(self, django_user): """ Create in Hive. Raise InstallException on failure. """ # Create table LOG.info('Creating table "%s"' % (self.name, )) try: # Already exists? tables = db_utils.meta_client().get_table("default", self.name) msg = _('Table "%(table)s" already exists.') % {'table': self.name} LOG.error(msg) raise InstallException(msg) except hive_metastore.ttypes.NoSuchObjectException: query_msg = _make_query_msg(self.hql) try: results = db_utils.execute_and_wait(django_user, query_msg) if not results: msg = _( 'Error creating table %(table)s: Operation timeout.' ) % { 'table': self.name } LOG.error(msg) raise InstallException(msg) except BeeswaxException, ex: msg = _('Error creating table %(table)s: %(error)s') % { 'table': self.name, 'error': ex } LOG.error(msg) raise InstallException(msg)
def describe_table(request, table): table_obj = db_utils.meta_client().get_table("default", table) # Show the first few rows hql = "SELECT * FROM `%s`" % (table,) query_msg = make_beeswax_query(request, hql) try: results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0) except: # Gracefully degrade if we're unable to load the results. logging.exception("Failed to read table '%s'" % table) results = None hdfs_link = location_to_url(request, table_obj.sd.location) load_form = beeswax.forms.LoadDataForm(table_obj) return render("describe_table.mako", request, dict( table=table_obj, table_name=table, top_rows=results and list(parse_results(results.data)) or None, hdfs_link=hdfs_link, load_form=load_form ))
def load(self, django_user): """ Load data into table. """ LOAD_HQL = \ """ LOAD DATA local INPATH '%(filename)s' OVERWRITE INTO TABLE %(tablename)s """ LOG.info('Loading data into table "%s"' % (self.name,)) hql = LOAD_HQL % dict(tablename=self.name, filename=self._contents_file) query_msg = _make_query_msg(hql) try: results = db_utils.execute_and_wait(django_user, query_msg) if not results: LOG.error('Error loading table %s: Operation timeout' % (self.name,)) return False except BeeswaxException, ex: LOG.error('Error loading table %s: %s' % (self.name, ex)) return False
def create(self, django_user): """ Create in Hive. Returns True/False. """ # Create table LOG.info('Creating table "%s"' % (self.name,)) try: # Already exists? tables = db_utils.meta_client().get_table("default", self.name) LOG.error('Table "%s" already exists' % (self.name,)) return False except hive_metastore.ttypes.NoSuchObjectException: query_msg = _make_query_msg(self.hql) try: results = db_utils.execute_and_wait(django_user, query_msg) if not results: LOG.error('Error creating table %s: Operation timeout' % (self.name,)) return False except BeeswaxException, ex: LOG.error('Error creating table %s: %s' % (self.name, ex)) return False return True
def load(self, django_user): """ Load data into table. Raise InstallException on failure. """ LOAD_HQL = \ """ LOAD DATA local INPATH '%(filename)s' OVERWRITE INTO TABLE %(tablename)s """ LOG.info('Loading data into table "%s"' % (self.name,)) hql = LOAD_HQL % dict(tablename=self.name, filename=self._contents_file) query_msg = _make_query_msg(hql) try: results = db_utils.execute_and_wait(django_user, query_msg) if not results: msg = _('Error loading table %(table)s: Operation timeout.') % {'table': self.name} LOG.error(msg) raise InstallException(msg) except BeeswaxException, ex: msg = _('Error loading table %(table)s: %(error)s') % {'table': self.name, 'error': ex} LOG.error(msg) raise InstallException(msg)
def create(self, django_user): """ Create in Hive. Raise InstallException on failure. """ # Create table LOG.info('Creating table "%s"' % (self.name,)) try: # Already exists? tables = db_utils.meta_client().get_table("default", self.name) msg = _('Table "%(table)s" already exists.') % {'table': self.name} LOG.error(msg) raise InstallException(msg) except hive_metastore.ttypes.NoSuchObjectException: query_msg = _make_query_msg(self.hql) try: results = db_utils.execute_and_wait(django_user, query_msg) if not results: msg = _('Error creating table %(table)s: Operation timeout.') % {'table': self.name} LOG.error(msg) raise InstallException(msg) except BeeswaxException, ex: msg = _('Error creating table %(table)s: %(error)s') % {'table': self.name, 'error': ex} LOG.error(msg) raise InstallException(msg)