示例#1
0
  def get_partitions(self, db_name, table, max_parts=None):
    if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
      max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()

    # DB name not supported in SHOW PARTITIONS
    self.use(db_name)

    return self.client.get_partitions(db_name, table.name, max_parts)
示例#2
0
文件: dbms.py 项目: jackesh/hue
  def get_partitions(self, db_name, table, max_parts=None):
    if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
      max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()

    # DB name not supported in SHOW PARTITIONS
    self.use(db_name)

    return self.client.get_partitions(db_name, table.name, max_parts)
示例#3
0
    def get_partitions(self,
                       db_name,
                       table,
                       partition_spec=None,
                       max_parts=None,
                       reverse_sort=True):
        if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get(
        ):
            max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()

        return self.client.get_partitions(db_name, table.name, partition_spec,
                                          max_parts, reverse_sort)
示例#4
0
文件: dbms.py 项目: rhmiller47/hue
  def get_sample(self, database, table, column=None, nested=None):
    result = None
    hql = None

    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

      if column or nested: # Could do column for any type, then nested with partitions 
        if self.server_name == 'impala':
          select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
          hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
      else:
        if table.partition_keys:  # Filter on max # of partitions for partitioned tables
          hql = self._get_sample_partition_query(database, table, limit)
        else:
          hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)

      if hql:
        query = hql_query(hql)
        handle = self.execute_and_wait(query, timeout_sec=5.0)

        if handle:
          result = self.fetch(handle, rows=100)
          self.close(handle)

    return result
示例#5
0
文件: dbms.py 项目: hdinsight/hue
  def get_sample(self, database, table, column=None, nested=None):
    result = None
    hql = None

    if not table.is_view:

      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

      if column or nested: # Could do column for any type, then nested with partitions 
        if self.server_name == 'impala':
          select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
          hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
      else:
        partition_query = ""
        if table.partition_keys:
          partitions = self.get_partitions(database, table, partition_spec=None, max_parts=1)
          partition_query = 'WHERE ' + ' AND '.join(["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[0].values)])
        hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_query, limit)

      if hql:
        query = hql_query(hql)
        handle = self.execute_and_wait(query, timeout_sec=5.0)

        if handle:
          result = self.fetch(handle, rows=100)
          self.close(handle)

    return result
示例#6
0
    def get_sample(self, database, table, column=None, nested=None):
        result = None
        hql = None

        if not table.is_view:
            limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

            if column or nested:  # Could do column for any type, then nested with partitions
                if self.server_name == 'impala':
                    select_clause, from_clause = ImpalaDbms.get_nested_select(
                        database, table.name, column, nested)
                    hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause,
                                                          from_clause, limit)
            else:
                if table.partition_keys:  # Filter on max # of partitions for partitioned tables
                    hql = self._get_sample_partition_query(
                        database, table, limit)
                else:
                    hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (
                        database, table.name, limit)

            if hql:
                query = hql_query(hql)
                handle = self.execute_and_wait(query, timeout_sec=5.0)

                if handle:
                    result = self.fetch(handle, rows=100)
                    self.close(handle)

        return result
示例#7
0
 def select_star_from(self, database, table):
   if table.partition_keys:  # Filter on max # of partitions for partitioned tables
     limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
     hql = self._get_sample_partition_query(database, table, limit)
   else:
     hql = "SELECT * FROM `%s`.`%s`" % (database, table.name)
   return self.execute_statement(hql)
示例#8
0
文件: dbms.py 项目: jackesh/hue
 def _get_browse_limit_clause(self, table):
   """Get the limit clause when browsing a partitioned table"""
   if table.partition_keys:
     limit = BROWSE_PARTITIONED_TABLE_LIMIT.get()
     if limit > 0:
       return "LIMIT %d" % (limit,)
   return ""
示例#9
0
文件: dbms.py 项目: rhmiller47/hue
 def select_star_from(self, database, table):
   if table.partition_keys:  # Filter on max # of partitions for partitioned tables
     limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
     hql = self._get_sample_partition_query(database, table, limit)
   else:
     hql = "SELECT * FROM `%s`.`%s`" % (database, table.name)
   return self.execute_statement(hql)
示例#10
0
 def _get_browse_limit_clause(self, table):
     """Get the limit clause when browsing a partitioned table"""
     if table.partition_keys:
         limit = BROWSE_PARTITIONED_TABLE_LIMIT.get()
         if limit > 0:
             return "LIMIT %d" % (limit, )
     return ""
示例#11
0
文件: tests.py 项目: shobull/hue
  def test_describe_partitioned_table_with_limit(self):
    # We have 2 partitions in the test table
    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("1")
    try:
      response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name)
      partition_values_json = json.loads(response.context['partition_values_json'])
      assert_equal(1, len(partition_values_json))
    finally:
      finish()

    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("3")
    try:
      response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name)
      partition_values_json = json.loads(response.context['partition_values_json'])
      assert_equal(2, len(partition_values_json))
    finally:
      finish()
示例#12
0
文件: tests.py 项目: agibsonccc/hue
 def test_browse_partitions_with_limit(self):
   # Limit to 90
   finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
   try:
     response = self.client.get("/catalog/table/default/test_partitions")
     assert_true("0x%x" % 89 in response.content, response.content)
     assert_false("0x%x" % 90 in response.content, response.content)
   finally:
     finish()
示例#13
0
文件: tests.py 项目: Lt-Pone/hue
 def test_browse_partitioned_table_with_limit(self):
   # Limit to 90
   finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
   try:
     response = self.client.get("/metastore/table/default/test_partitions")
     assert_true("0x%x" % 89 in response.content, response.content)
     assert_false("0x%x" % 90 in response.content, response.content)
   finally:
     finish()
示例#14
0
 def test_describe_partitioned_table_with_limit(self):
   # Limit to 90
   finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
   try:
     response = self.client.get("/metastore/table/%s/test_partitions" % self.db_name)
     assert_true("0x%x" % 89 in response.content, response.content)
     assert_false("0x%x" % 90 in response.content, response.content)
   finally:
     finish()
示例#15
0
  def get_sample(self, database, table):
    """No samples if it's a view (HUE-526)"""
    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
      hql = "SELECT * FROM `%s.%s` LIMIT %s" % (database, table.name, limit)
      query = hql_query(hql)
      handle = self.execute_and_wait(query, timeout_sec=5.0)

      if handle:
        return self.fetch(handle)
示例#16
0
文件: dbms.py 项目: jackesh/hue
  def get_sample(self, database, table):
    """No samples if it's a view (HUE-526)"""
    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
      hql = "SELECT * FROM `%s.%s` LIMIT %s" % (database, table.name, limit)
      query = hql_query(hql)
      handle = self.execute_and_wait(query, timeout_sec=5.0)

      if handle:
        return self.fetch(handle)
示例#17
0
  def test_describe_partitioned_table_with_limit(self):
    if is_live_cluster():
      raise SkipTest('HUE-2902: Test is not re-entrant')

    # Limit to 90
    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
    try:
      response = self.client.get("/metastore/table/%s/test_partitions" % self.db_name)
      assert_true("0x%x" % 89 in response.content, response.content)
      assert_false("0x%x" % 90 in response.content, response.content)
    finally:
      finish()
示例#18
0
文件: tests.py 项目: neiodavince/hue
  def test_describe_partitioned_table_with_limit(self):
    if is_live_cluster():
      raise SkipTest('HUE-2902: Test is not re-entrant')

    # Limit to 90
    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
    try:
      response = self.client.get("/metastore/table/%s/test_partitions" % self.db_name)
      assert_true("0x%x" % 89 in response.content, response.content)
      assert_false("0x%x" % 90 in response.content, response.content)
    finally:
      finish()
示例#19
0
文件: tests.py 项目: erickt/hue
  def test_browse_limit_overrides_partition_limit(self):
    # Limit to 45
    finish = [
        BROWSE_TABLE_LIMIT.set_for_testing("45"),
        BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90"),
    ]

    try:
      response = self.client.get("/metastore/table/default/test_partitions")
      assert_true("0x%x" % 44 in response.content, response.content)
      assert_false("0x%x" % 45 in response.content, response.content)
    finally:
      for f in finish:
        f()
示例#20
0
  def get_browse_partition_clause(self, table, partitions):
    """Get the where clause to limit reading data in the first available partitions"""
    if partitions and BROWSE_PARTITIONED_TABLE_LIMIT.get() > 0:
      partition_values = partitions[0].values
      partition_keys = table.partition_keys
      partition_dict = zip(partition_keys, partition_values)

      fields = []
      for key, value in partition_dict:
          if key.type == "string":
            fields.append("`%s` = '%s'" % (key.name, value))
          else:
            fields.append("`%s` = %s" % (key.name, value))

      return "WHERE " + " AND ".join(fields)
示例#21
0
    def get_browse_partition_clause(self, table, partitions):
        """Get the where clause to limit reading data in the first available partitions"""
        if partitions and BROWSE_PARTITIONED_TABLE_LIMIT.get() > 0:
            partition_values = partitions[0].values
            partition_keys = table.partition_keys
            partition_dict = zip(partition_keys, partition_values)

            fields = []
            for key, value in partition_dict:
                if key.type == "string":
                    fields.append("`%s` = '%s'" % (key.name, value))
                else:
                    fields.append("`%s` = %s" % (key.name, value))

            return "WHERE " + " AND ".join(fields)
示例#22
0
文件: dbms.py 项目: erickt/hue
  def get_sample(self, database, table):
    """No samples if it's a view (HUE-526)"""
    if not table.is_view:
      limit = BROWSE_TABLE_LIMIT.get()
      if not limit:
        limit = 100

      limit = min(limit, BROWSE_PARTITIONED_TABLE_LIMIT.get())
      hql = "SELECT * FROM %s.%s LIMIT %s" % (database, table.name, limit)
      query = hql_query(hql)
      handle = self.execute_and_wait(query, timeout_sec=5.0)

      if handle:
        result = self.fetch(handle, rows=limit)
        self.close(handle)
        return result
示例#23
0
  def get_sample(self, database, table):
    """No samples if it's a view (HUE-526)"""
    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
      partition_query = ""
      if table.partition_keys:
        partitions = self.get_partitions(database, table, 1)
        partition_query = 'WHERE ' + ' AND '.join(["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[0].values)])
      hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_query, limit)
      query = hql_query(hql)
      handle = self.execute_and_wait(query, timeout_sec=5.0)

      if handle:
        result = self.fetch(handle, rows=100)
        self.close(handle)
        return result
示例#24
0
文件: dbms.py 项目: cloudnautique/hue
  def get_sample(self, database, table):
    """No samples if it's a view (HUE-526)"""
    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
      partition_query = ""
      if table.partition_keys:
        partitions = self.get_partitions(database, table, partition_spec=None, max_parts=1)
        partition_query = 'WHERE ' + ' AND '.join(["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[0].values)])
      hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_query, limit)
      query = hql_query(hql)
      handle = self.execute_and_wait(query, timeout_sec=5.0)

      if handle:
        result = self.fetch(handle, rows=100)
        self.close(handle)
        return result
示例#25
0
  def get_sample(self, database, table, column=None, nested=None):
    result = None
    hql = None

    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

      if table.partition_keys:  # Filter on max # of partitions for partitioned tables
        hql = self._get_sample_partition_query(database, table, limit)
      else:
        hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)

      if hql:
        query = hql_query(hql)
        handle = self.execute_and_wait(query, timeout_sec=5.0)

        if handle:
          result = self.fetch(handle, rows=100)
          self.close(handle)

    return result
示例#26
0
文件: dbms.py 项目: shobull/hue
  def get_sample(self, database, table, column=None, nested=None):
    result = None
    hql = None

    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

      if table.partition_keys:  # Filter on max # of partitions for partitioned tables
        hql = self._get_sample_partition_query(database, table, limit)
      else:
        hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)

      if hql:
        query = hql_query(hql)
        handle = self.execute_and_wait(query, timeout_sec=5.0)

        if handle:
          result = self.fetch(handle, rows=100)
          self.close(handle)

    return result
示例#27
0
文件: dbms.py 项目: hdinsight/hue
    def get_sample(self, database, table, column=None, nested=None):
        result = None
        hql = None

        if not table.is_view:

            limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

            if column or nested:  # Could do column for any type, then nested with partitions
                if self.server_name == 'impala':
                    select_clause, from_clause = ImpalaDbms.get_nested_select(
                        database, table.name, column, nested)
                    hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause,
                                                          from_clause, limit)
            else:
                partition_query = ""
                if table.partition_keys:
                    partitions = self.get_partitions(database,
                                                     table,
                                                     partition_spec=None,
                                                     max_parts=1)
                    partition_query = 'WHERE ' + ' AND '.join([
                        "%s='%s'" % (table.partition_keys[idx].name, key)
                        for idx, key in enumerate(partitions[0].values)
                    ])
                hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (
                    database, table.name, partition_query, limit)

            if hql:
                query = hql_query(hql)
                handle = self.execute_and_wait(query, timeout_sec=5.0)

                if handle:
                    result = self.fetch(handle, rows=100)
                    self.close(handle)

        return result
示例#28
0
  def get_partitions(self, db_name, table, max_parts=None):
    if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
      max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()

    return self.client.get_partitions(db_name, table.name, max_parts)
示例#29
0
文件: dbms.py 项目: icheckmate/hue
  def get_partitions(self, db_name, table, max_parts=None):
    if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
      max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()

    return self.client.get_partitions(db_name, table.name, max_parts)
示例#30
0
文件: dbms.py 项目: cloudnautique/hue
  def get_partitions(self, db_name, table, partition_spec=None, max_parts=None, reverse_sort=True):
    if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
      max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()

    return self.client.get_partitions(db_name, table.name, partition_spec, max_parts, reverse_sort)