def load_after_create(request, database): """ Automatically load data into a newly created table. We get here from the create's on_success_url, and expect to find ``table`` and ``path`` from the parameters. """ tablename = request.REQUEST.get("table") path = request.REQUEST.get("path") is_remove_header = request.REQUEST.get("removeHeader").lower() == "on" and not path.endswith("gz") if not tablename or not path: msg = _("Internal error: Missing needed parameter to load data into table.") LOG.error(msg) raise PopupException(msg) if is_remove_header: remove_header(request.fs, path) LOG.debug("Auto loading data from %s into table %s" % (path, tablename)) hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, tablename) query = hql_query(hql) on_success_url = reverse("metastore:describe_table", kwargs={"database": database, "table": tablename}) return execute_directly(request, query, on_success_url=on_success_url)
def load_after_create(request, database): """ Automatically load data into a newly created table. We get here from the create's on_success_url, and expect to find ``table`` and ``path`` from the parameters. """ tablename = request.REQUEST.get('table') path = request.REQUEST.get('path') is_remove_header = request.REQUEST.get( 'removeHeader').lower() == 'on' and not path.endswith('gz') if not tablename or not path: msg = _( 'Internal error: Missing needed parameter to load data into table.' ) LOG.error(msg) raise PopupException(msg) if is_remove_header: remove_header(request.fs, path) LOG.debug("Auto loading data from %s into table %s" % (path, tablename)) hql = "LOAD DATA INPATH '%s' INTO TABLE `%s.%s`" % (path, database, tablename) query = hql_query(hql) on_success_url = reverse('metastore:describe_table', kwargs={ 'database': database, 'table': tablename }) return execute_directly(request, query, on_success_url=on_success_url)
def load_after_create(request, database): """ Automatically load data into a newly created table. We get here from the create's on_success_url, and expect to find ``table`` and ``path`` from the parameters. """ tablename = request.REQUEST.get('table') path = request.REQUEST.get('path') is_remove_header = request.REQUEST.get( 'removeHeader').lower() == 'on' and not path.endswith('gz') if not tablename or not path: msg = _( 'Internal error: Missing needed parameter to load data into table.' ) LOG.error(msg) raise PopupException(msg) if is_remove_header: try: remove_header(request.fs, path) except Exception, e: raise PopupException( _("The headers of the file could not be removed."), detail=e)
def load_after_create(request, database): """ Automatically load data into a newly created table. We get here from the create's on_success_url, and expect to find ``table`` and ``path`` from the parameters. """ tablename = request.REQUEST.get('table') path = request.REQUEST.get('path') is_remove_header = request.REQUEST.get('removeHeader').lower() == 'on' and not path.endswith('gz') if not tablename or not path: msg = _('Internal error: Missing needed parameter to load data into table.') LOG.error(msg) raise PopupException(msg) if is_remove_header: try: remove_header(request.fs, path) except Exception, e: raise PopupException(_("The headers of the file could not be removed."), detail=e)
def test_remove_header(self): fs = self.cluster.fs path = "/tmp/test_remove_header.txt" data_header = "destination\trank" data_body = """thailand\t10 costarica\t? curacao\t?""" data = data_header + '\n' + data_body f = fs.open(path, "w") f.write("hello") f.close() encoding = i18n.get_site_encoding() do_overwrite_save(fs, path, data.encode(encoding)) assert_not_equal(data_body, fs.open(path).read()) remove_header(fs, path) assert_equals(data_body, fs.open(path).read())