Ejemplo n.º 1
0
    def __init__(self, host=HOST, port=PORT, url=None, user=USER, password=PASSWORD, secure=False, \
                 verify_ssl=True, content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT,
                 retries=RETRIES, retry_delay=RETRY_DELAY):
        """
        Creates a new connection instance.

        See docs above.
        """
        if not content_type in CONTENT_TYPES:
            raise ImproperlyConfigured(
                _("Invalid ``content_type`` {0} value.".format(content_type)))

        if not host:
            raise ImproperlyConfigured(
                _("Invalid ``host`` {0} value.".format(host)))

        if not port:
            raise ImproperlyConfigured(
                _("Invalid ``port`` {0} value.".format(port)))

        self.host = host
        self.port = port
        self.user = user
        self.password = password
        self.secure = secure
        self.verify_ssl = verify_ssl
        self.content_type = CONTENT_TYPES_DICT[content_type]
        self.perfect_dict = perfect_dict
        self.retries = retries
        self.retry_delay = retry_delay
        self.__connect(url)
Ejemplo n.º 2
0
    def __init__(self, host=HOST, port=PORT, url=None, user=USER, password=PASSWORD, secure=False, \
                 content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT,
                 retries=RETRIES, retry_delay=RETRY_DELAY):
        """
        Creates a new connection instance.

        See docs above.
        """
        if not content_type in CONTENT_TYPES:
            raise ImproperlyConfigured(_("Invalid ``content_type`` {0} value.".format(content_type)))

        if not host:
            raise ImproperlyConfigured(_("Invalid ``host`` {0} value.".format(host)))

        if not port:
            raise ImproperlyConfigured(_("Invalid ``port`` {0} value.".format(port)))

        self.host = host
        self.port = port
        self.user = user
        self.password = password
        self.secure = secure
        self.content_type = CONTENT_TYPES_DICT[content_type]
        self.perfect_dict = perfect_dict
        self.retries = retries
        self.retry_delay = retry_delay
        self.__connect(url)
Ejemplo n.º 3
0
    def __init__(self, host=HOST, port=PORT, user=USER, password=PASSWORD, secure=False, \
                 content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT):
        """
        Creates a new connection instance.

        See docs above.
        """
        if not content_type in CONTENT_TYPES:
            raise ImproperlyConfigured(
                _("Invalid ``content_type`` {0} value.".format(content_type)))

        if not host:
            raise ImproperlyConfigured(
                _("Invalid ``host`` {0} value.".format(host)))

        if not port:
            raise ImproperlyConfigured(
                _("Invalid ``port`` {0} value.".format(port)))

        self.host = host
        self.port = port
        self.user = user
        self.password = password
        self.secure = secure
        self.content_type = CONTENT_TYPES_DICT[content_type]
        self.perfect_dict = perfect_dict
        self.__connect()
Ejemplo n.º 4
0
    def fetch_all_rows(self, with_row_id=False, raw=False, perfect_dict=None,
                       flat=False, filter_string=None, scanner_config={},
                       fail_silently=True):
        """Fetch all table rows.

        :param bool with_row_id: If set to True, returned along with row id.
        :param bool raw: If set to True, raw response is returned.
        :param bool perfect_dict: If set to True, a perfect dict structure is
            used for output data.
        :param bool flat:
        :param string filter_string: If set, applies the given filter string
            to the scanner.
        :param dict scanner_config:
        :param mixed flat_silently.
        :return list:

        :example:
        >>> filter_string = '{"type": "RowFilter", "op": "EQUAL", "comparator": '
        >>>                 '{"type": "RegexStringComparator", "value": "^row_1.+"}}'
        >>> rows = self.table.fetch_all_rows(
        >>>     with_row_id = True,
        >>>     perfect_dict = perfect_dict,
        >>>     filter_string = row_filter_string
        >>>     )
        """
        if self.check_if_exists_on_scanner_operations:
            if not self.exists(fail_silently=fail_silently):
                return None

        if perfect_dict is None:
            perfect_dict = self.connection.perfect_dict

        try:
            scanner = self._scanner(filter_string=filter_string,
                                    data=scanner_config,
                                    fail_silently=fail_silently)
        except HTTPError as e:
            if fail_silently:
                return []
            raise DoesNotExist(_("""Table "{0}" does not exist."""
                               "".format(self.name)))

        if scanner:
            res = scanner.results(perfect_dict=perfect_dict,
                                  with_row_id=with_row_id,
                                  raw=raw)
            scanner.delete ()
        else:
            if fail_silently:
                return None
            raise DoesNotExist(_("""Table "{0}" does not exist.""".format(self.name)))

        if flat:
            res = list(res)

        return res
Ejemplo n.º 5
0
    def fetch_all_rows(self, with_row_id=False, raw=False, perfect_dict=None, flat=False, filter_string=None, \
                       scanner_config='', fail_silently=True):
        """
        Fetches all table rows.

        :param bool with_row_id: If set to True, returned along with row id.
        :param bool raw: If set to True, raw response is returned.
        :param bool perfect_dict: If set to True, a perfect dict struture is used for output data.
        :param string filter_string: If set, applies the given filter string to the scanner.
        :returns list:

        :example:
        >>> filter_string = '{"type": "RowFilter", "op": "EQUAL", "comparator": '
        >>>                 '{"type": "RegexStringComparator", "value": "^row_1.+"}}'
        >>> rows = self.table.fetch_all_rows(
        >>>     with_row_id = True,
        >>>     perfect_dict = perfect_dict,
        >>>     filter_string = row_filter_string
        >>>     )
        """
        if self.check_if_exists_on_scanner_operations:
            if not self.exists(fail_silently=fail_silently):
                return None

        if perfect_dict is None:
            perfect_dict = self.connection.perfect_dict

        try:
            scanner = self._scanner(filter_string=filter_string,
                                    data=scanner_config,
                                    fail_silently=fail_silently)
        except HTTPError as e:
            if fail_silently:
                return []
            raise DoesNotExist(
                _("""Table "{0}" does not exist.""".format(self.name)))

        if scanner:
            res = scanner.results(perfect_dict=perfect_dict,
                                  with_row_id=with_row_id,
                                  raw=raw)
            scanner.delete()
        else:
            if fail_silently:
                return None
            raise DoesNotExist(
                _("""Table "{0}" does not exist.""".format(self.name)))

        if flat:
            res = list(res)

        return res
Ejemplo n.º 6
0
    def __init__(self, connection, url='', data={}, decode_content=False, \
                 method=DEFAULT_METHOD, fail_silently=True):
        """
        See the docs above.
        """
        assert method in METHODS

        self.__connection = connection
        self.url = url
        self.data = data
        self.decode_content = decode_content
        self.fail_silently = fail_silently
        self.verify_ssl = connection.verify_ssl

        if not self.verify_ssl:
            requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

        headers = {
            'Accept': str(self.__connection.content_type),
            'Content-type': str(self.__connection.content_type) + \
                            '; charset=UTF-8'
            }
        endpoint_url = self.__connection.base_url + self.url

        is_xml_request = False
        if isinstance(self.data, string_types) and self.data[:1] == '<':
            is_xml_request = True

        if is_xml_request:
            headers['Content-type'] = 'text/xml'
        else:
            data = json.dumps(data)

        request_data = {'url': endpoint_url, 'headers': headers}

        if DELETE != method:
            request_data['data'] = data

        if self.__connection.user and self.__connection.password:
            request_data['auth'] = HTTPBasicAuth(self.__connection.user,
                                                 self.__connection.password)

        for i in range(connection.retries + 1):
            self.response = self.call(method, request_data)

            if self.response.status_code in status_codes.BAD_STATUS_CODES:
                # If status code is 4** or 5** or raise an exception
                # means, retry.
                if i < connection.retries:
                    delay = connection.retry_delay * (2**i)
                    logger.warn(
                        _("Hbase returned error: {0}. Sleeping "
                          "for {1} seconds").format(self.response.reason,
                                                    delay))
                    time.sleep(delay)
                else:
                    return None
            else:
                return None
Ejemplo n.º 7
0
    def __init__(self, connection, url='', data={}, decode_content=False, \
                 method=DEFAULT_METHOD, fail_silently=True):
        """
        See the docs above.
        """
        assert method in METHODS

        self.__connection = connection
        self.url = url
        self.data = data
        self.decode_content = decode_content
        self.fail_silently = fail_silently
        headers = {
            'Accept': str(self.__connection.content_type),
            'Content-type': str(self.__connection.content_type) + \
                            '; charset=UTF-8'
            }
        endpoint_url = self.__connection.base_url + self.url

        is_xml_request = False
        if isinstance(self.data, string_types) and self.data[:1] == '<':
            is_xml_request = True

        if is_xml_request:
            headers['Content-type'] = 'text/xml'
        else:
            data = json.dumps(data)

        request_data = {
            'url': endpoint_url,
            'headers': headers
        }

        if DELETE != method:
            request_data['data'] = data

        if self.__connection.user and self.__connection.password:
            request_data['auth'] = HTTPBasicAuth(
                self.__connection.user,
                self.__connection.password
                )

        for i in range(connection.retries + 1):
            self.response = self.call(method, request_data)

            if self.response.status_code in status_codes.BAD_STATUS_CODES:
                # If status code is 4** or 5** or raise an exception
                # means, retry.
                if i < connection.retries:
                    delay = connection.retry_delay * (2**i)
                    logger.warn(
                        _("Hbase returned error: {0}. Sleeping "
                          "for {1} seconds").format(self.response.reason, delay)
                        )
                    time.sleep(delay)
                else:
                    return None
            else:
                return None
Ejemplo n.º 8
0
    def _extract_cell_data(cell_data, perfect_dict=PERFECT_DICT):
        """Extract the cell data.

        :param list|dict cell_data:
        :param bool perfect_dict: If set to True, a perfect dictionary is
            returned (see data structure #2 and #4). Otherwise, ordinary
            structure is returned (see data structure #1 and #3). If not given,
            global setting is used.
        :return list|dict:
        """
        assert isinstance(cell_data, (list, dict))

        extracted_cell_data = None

        # Single column
        if isinstance(cell_data, dict):
            extracted_cell_data = Table._extract_column_data(
                cell_data, perfect_dict=perfect_dict)

        # Multiple column
        else:
            if perfect_dict:
                extracted_cell_data = {}
                for column_data in cell_data:
                    d = Table._extract_column_data(column_data,
                                                   perfect_dict=perfect_dict)
                    for d_key, d_val in d.items():
                        if PY3:
                            if isinstance(d_val, bytes):
                                d_val = d_val.decode('utf8')

                        if d_key in extracted_cell_data:
                            overlap = set(
                                extracted_cell_data[d_key].keys()) & set(
                                    d_val.keys())
                            if overlap:
                                if 1 == len(overlap):
                                    plural = ''
                                    keys = overlap.pop()
                                else:
                                    plural = 's'
                                    keys = '[{}]'.format(','.join(
                                        list(overlap)))
                                logger.error(
                                    _("Was just about to lose overlapping "
                                      "data for key{0} {1}:{2}"
                                      "").format(plural, d_key, keys))
                            else:
                                extracted_cell_data[d_key].update(d_val)
                        else:
                            extracted_cell_data[d_key] = d_val
            else:
                extracted_cell_data = {}
                for column_data in cell_data:
                    extracted_cell_data.update(
                        Table._extract_column_data(column_data,
                                                   perfect_dict=perfect_dict))
        return extracted_cell_data
Ejemplo n.º 9
0
    def _extract_cell_data(cell_data, perfect_dict=PERFECT_DICT):
        """
        Extracts the cell data.

        :param list|dict cell_data:
        :param bool perfect_dict: If set to True, a perfect dictionary is returned (see data structure #2 and #4).
            Otherwise, ordinary structure is returned (see data structure #1 and #3). If not given, global setting
            is used.
        :return list|dict:
        """
        assert isinstance(cell_data, (list, dict))

        extracted_cell_data = None

        # Single column
        if isinstance(cell_data, dict):
            extracted_cell_data = Table._extract_column_data(cell_data, perfect_dict=perfect_dict)

        # Multiple column
        else:
            if perfect_dict:
                extracted_cell_data = {}
                for column_data in cell_data:
                    d = Table._extract_column_data(column_data, perfect_dict=perfect_dict)
                    for d_key, d_val in d.items():
                        if PY3:
                            if isinstance(d_val, bytes):
                                d_val = d_val.decode('utf8')

                        if d_key in extracted_cell_data:
                            overlap = set(extracted_cell_data[d_key].keys()) & set(d_val.keys())
                            if overlap:
                                if 1 == len(overlap):
                                    plural = ''
                                    keys = overlap.pop()
                                else:
                                    plural = 's'
                                    keys = '[{}]'.format(','.join(list(overlap)))
                                logger.error(_("Was just about to lose overlapping data "
                                               "for key{0} {1}:{2}").format(plural, d_key, keys))
                            else:
                                extracted_cell_data[d_key].update(d_val)
                        else:
                            extracted_cell_data[d_key] = d_val
            else:
                extracted_cell_data = {}
                for column_data in cell_data:
                    extracted_cell_data.update(Table._extract_column_data(column_data, perfect_dict=perfect_dict))
        return extracted_cell_data
Ejemplo n.º 10
0
    def _build_delete_url(self, row, column=None, qualifier=None):
        """
        Builds a URL to use for sending the DELETE commands.

        :param str row: Row id to delete.
        :param str column: Column
        :param str qualifier: Column qualifier.
        :return str:
        """
        if qualifier and not column:
            raise InvalidArguments(_("Qualifier can't be given without column."))

        # Base URL
        parts = []
        parts.append("{table_name}/{row}".format(table_name=self.name, row=row))

        if qualifier:
            parts.append("{0}:{1}".format(column, qualifier))
        elif column:
            parts.append("{0}".format(column))

        return '/'.join(parts)
Ejemplo n.º 11
0
    def _build_delete_url(self, row, column=None, qualifier=None):
        """
        Builds a URL to use for sending the DELETE commands.

        :param str row: Row id to delete.
        :param str column: Column
        :param str qualifier: Column qualifier.
        :return str:
        """
        if qualifier and not column:
            raise InvalidArguments(
                _("Qualifier can't be given without column."))

        # Base URL
        parts = []
        parts.append("{table_name}/{row}".format(table_name=self.name,
                                                 row=row))

        if qualifier:
            parts.append("{0}:{1}".format(column, qualifier))
        elif column:
            parts.append("{0}".format(column))

        return '/'.join(parts)
Ejemplo n.º 12
0
    def _get(self,
             row,
             columns=None,
             timestamp=None,
             decode_content=True,
             number_of_versions=None,
             raw=False,
             perfect_dict=None,
             fail_silently=True):
        """Retrieve one or more cells from a full row.

        Retrieve one or more cells from a full row or one or more specified
        columns in the row, with optional filtering via timestamp, and an
        optional restriction on the maximum number of versions to return.

        The `raw` argument is dominant. If given, the raw response it returned.
        Otherwise, a nice response is returned that does make sense.
        If `perfect_dict` set to True, then we return a nice dict, instead of a
        horrible one.

        In result JSON, the value of the `$` field (key) is the cell data.

        :param str row:
        :param list|set|tuple|dict columns:
        :param timestamp: Not yet used.
        :param bool decode_content: If set to True, content is decoded using
            ``stargate.json_decoder.json_decode``.
        :param int number_of_versions: If provided, multiple versions of the
            given record are returned.
        :param bool perfect_dict:
        :param bool raw:
        :return dict:
        """
        if self.check_if_exists_on_row_fetch:
            if not self.exists(fail_silently=fail_silently):
                return None

        if perfect_dict is None:
            perfect_dict = self.connection.perfect_dict

        # If just one column given as string, make a list of it.
        if isinstance(columns, string_types):
            columns = [columns]

        # Base URL
        url = "{table_name}/{row}/".format(table_name=self.name, row=row)

        url += self._build_url_parts(columns)

        # Handling the timestamps
        if timestamp:
            if isinstance(timestamp, str):
                timestamp = int(timestamp)
            url += "/{timestamp},{timestamp_plus}/".format(
                timestamp=str(timestamp), timestamp_plus=str(timestamp + 1))

        # If should be versioned, adding additional URL parts.
        if number_of_versions is not None:
            assert isinstance(number_of_versions, int)
            url += '?v={0}'.format(str(number_of_versions))

        response = HttpRequest(connection=self.connection,
                               url=url,
                               decode_content=decode_content,
                               fail_silently=fail_silently).get_response()

        response_content = response.content

        if raw:
            return response_content

        if response_content:
            try:
                res = Table._extract_usable_data(response_content,
                                                 perfect_dict=perfect_dict,
                                                 with_row_id=False)
                if isinstance(res, (list, tuple)) and 1 == len(res):
                    return res[0]
                if not fail_silently:
                    raise ParseError(
                        _("No usable data found in HTTP response."))
            except Exception as e:
                if not fail_silently:
                    raise ParseError(
                        _("Failed to parse the HTTP response. "
                          "Error details: {0}").format(str(e)))
Ejemplo n.º 13
0
    def _get(self, row, columns=None, timestamp=None, decode_content=True,
             number_of_versions=None, raw=False,  perfect_dict=None,
             fail_silently=True):
        """Retrieve one or more cells from a full row.

        Retrieve one or more cells from a full row or one or more specified
        columns in the row, with optional filtering via timestamp, and an
        optional restriction on the maximum number of versions to return.

        The `raw` argument is dominant. If given, the raw response it returned.
        Otherwise, a nice response is returned that does make sense.
        If `perfect_dict` set to True, then we return a nice dict, instead of a
        horrible one.

        In result JSON, the value of the `$` field (key) is the cell data.

        :param str row:
        :param list|set|tuple|dict columns:
        :param timestamp: Not yet used.
        :param bool decode_content: If set to True, content is decoded using
            ``stargate.json_decoder.json_decode``.
        :param int number_of_versions: If provided, multiple versions of the
            given record are returned.
        :param bool perfect_dict:
        :param bool raw:
        :return dict:
        """
        if self.check_if_exists_on_row_fetch:
            if not self.exists(fail_silently=fail_silently):
                return None

        if perfect_dict is None:
            perfect_dict = self.connection.perfect_dict

        # If just one column given as string, make a list of it.
        if isinstance(columns, string_types):
            columns = [columns]

        # Base URL
        url = "{table_name}/{row}/".format(table_name=self.name, row=row)

        url += self._build_url_parts(columns)

        # Handling the timestamps
        if timestamp:
            if isinstance(timestamp, str):
                timestamp = int(timestamp)
            url += "/{timestamp},{timestamp_plus}/".format(
                timestamp = str(timestamp),
                timestamp_plus = str(timestamp + 1)
                )

        # If should be versioned, adding additional URL parts.
        if number_of_versions is not None:
            assert isinstance(number_of_versions, int)
            url += '?v={0}'.format(str(number_of_versions))

        response = HttpRequest(
            connection = self.connection,
            url = url,
            decode_content = decode_content,
            fail_silently = fail_silently
            ).get_response()

        response_content = response.content

        if raw:
            return response_content

        if response_content:
            try:
                res = Table._extract_usable_data(response_content,
                                                 perfect_dict=perfect_dict,
                                                 with_row_id=False)
                if isinstance(res, (list, tuple)) and 1 == len(res):
                    return res[0]
                if not fail_silently:
                    raise ParseError(_("No usable data found in HTTP response."))
            except Exception as e:
                if not fail_silently:
                    raise ParseError(_("Failed to parse the HTTP response. "
                                       "Error details: {0}").format(str(e)))