def __init__(self, host=HOST, port=PORT, url=None, user=USER, password=PASSWORD, secure=False, \ verify_ssl=True, content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT, retries=RETRIES, retry_delay=RETRY_DELAY): """ Creates a new connection instance. See docs above. """ if not content_type in CONTENT_TYPES: raise ImproperlyConfigured( _("Invalid ``content_type`` {0} value.".format(content_type))) if not host: raise ImproperlyConfigured( _("Invalid ``host`` {0} value.".format(host))) if not port: raise ImproperlyConfigured( _("Invalid ``port`` {0} value.".format(port))) self.host = host self.port = port self.user = user self.password = password self.secure = secure self.verify_ssl = verify_ssl self.content_type = CONTENT_TYPES_DICT[content_type] self.perfect_dict = perfect_dict self.retries = retries self.retry_delay = retry_delay self.__connect(url)
def __init__(self, host=HOST, port=PORT, url=None, user=USER, password=PASSWORD, secure=False, \ content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT, retries=RETRIES, retry_delay=RETRY_DELAY): """ Creates a new connection instance. See docs above. """ if not content_type in CONTENT_TYPES: raise ImproperlyConfigured(_("Invalid ``content_type`` {0} value.".format(content_type))) if not host: raise ImproperlyConfigured(_("Invalid ``host`` {0} value.".format(host))) if not port: raise ImproperlyConfigured(_("Invalid ``port`` {0} value.".format(port))) self.host = host self.port = port self.user = user self.password = password self.secure = secure self.content_type = CONTENT_TYPES_DICT[content_type] self.perfect_dict = perfect_dict self.retries = retries self.retry_delay = retry_delay self.__connect(url)
def __init__(self, host=HOST, port=PORT, user=USER, password=PASSWORD, secure=False, \ content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT): """ Creates a new connection instance. See docs above. """ if not content_type in CONTENT_TYPES: raise ImproperlyConfigured( _("Invalid ``content_type`` {0} value.".format(content_type))) if not host: raise ImproperlyConfigured( _("Invalid ``host`` {0} value.".format(host))) if not port: raise ImproperlyConfigured( _("Invalid ``port`` {0} value.".format(port))) self.host = host self.port = port self.user = user self.password = password self.secure = secure self.content_type = CONTENT_TYPES_DICT[content_type] self.perfect_dict = perfect_dict self.__connect()
def fetch_all_rows(self, with_row_id=False, raw=False, perfect_dict=None, flat=False, filter_string=None, scanner_config={}, fail_silently=True): """Fetch all table rows. :param bool with_row_id: If set to True, returned along with row id. :param bool raw: If set to True, raw response is returned. :param bool perfect_dict: If set to True, a perfect dict structure is used for output data. :param bool flat: :param string filter_string: If set, applies the given filter string to the scanner. :param dict scanner_config: :param mixed flat_silently. :return list: :example: >>> filter_string = '{"type": "RowFilter", "op": "EQUAL", "comparator": ' >>> '{"type": "RegexStringComparator", "value": "^row_1.+"}}' >>> rows = self.table.fetch_all_rows( >>> with_row_id = True, >>> perfect_dict = perfect_dict, >>> filter_string = row_filter_string >>> ) """ if self.check_if_exists_on_scanner_operations: if not self.exists(fail_silently=fail_silently): return None if perfect_dict is None: perfect_dict = self.connection.perfect_dict try: scanner = self._scanner(filter_string=filter_string, data=scanner_config, fail_silently=fail_silently) except HTTPError as e: if fail_silently: return [] raise DoesNotExist(_("""Table "{0}" does not exist.""" "".format(self.name))) if scanner: res = scanner.results(perfect_dict=perfect_dict, with_row_id=with_row_id, raw=raw) scanner.delete () else: if fail_silently: return None raise DoesNotExist(_("""Table "{0}" does not exist.""".format(self.name))) if flat: res = list(res) return res
def fetch_all_rows(self, with_row_id=False, raw=False, perfect_dict=None, flat=False, filter_string=None, \ scanner_config='', fail_silently=True): """ Fetches all table rows. :param bool with_row_id: If set to True, returned along with row id. :param bool raw: If set to True, raw response is returned. :param bool perfect_dict: If set to True, a perfect dict struture is used for output data. :param string filter_string: If set, applies the given filter string to the scanner. :returns list: :example: >>> filter_string = '{"type": "RowFilter", "op": "EQUAL", "comparator": ' >>> '{"type": "RegexStringComparator", "value": "^row_1.+"}}' >>> rows = self.table.fetch_all_rows( >>> with_row_id = True, >>> perfect_dict = perfect_dict, >>> filter_string = row_filter_string >>> ) """ if self.check_if_exists_on_scanner_operations: if not self.exists(fail_silently=fail_silently): return None if perfect_dict is None: perfect_dict = self.connection.perfect_dict try: scanner = self._scanner(filter_string=filter_string, data=scanner_config, fail_silently=fail_silently) except HTTPError as e: if fail_silently: return [] raise DoesNotExist( _("""Table "{0}" does not exist.""".format(self.name))) if scanner: res = scanner.results(perfect_dict=perfect_dict, with_row_id=with_row_id, raw=raw) scanner.delete() else: if fail_silently: return None raise DoesNotExist( _("""Table "{0}" does not exist.""".format(self.name))) if flat: res = list(res) return res
def __init__(self, connection, url='', data={}, decode_content=False, \ method=DEFAULT_METHOD, fail_silently=True): """ See the docs above. """ assert method in METHODS self.__connection = connection self.url = url self.data = data self.decode_content = decode_content self.fail_silently = fail_silently self.verify_ssl = connection.verify_ssl if not self.verify_ssl: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) headers = { 'Accept': str(self.__connection.content_type), 'Content-type': str(self.__connection.content_type) + \ '; charset=UTF-8' } endpoint_url = self.__connection.base_url + self.url is_xml_request = False if isinstance(self.data, string_types) and self.data[:1] == '<': is_xml_request = True if is_xml_request: headers['Content-type'] = 'text/xml' else: data = json.dumps(data) request_data = {'url': endpoint_url, 'headers': headers} if DELETE != method: request_data['data'] = data if self.__connection.user and self.__connection.password: request_data['auth'] = HTTPBasicAuth(self.__connection.user, self.__connection.password) for i in range(connection.retries + 1): self.response = self.call(method, request_data) if self.response.status_code in status_codes.BAD_STATUS_CODES: # If status code is 4** or 5** or raise an exception # means, retry. if i < connection.retries: delay = connection.retry_delay * (2**i) logger.warn( _("Hbase returned error: {0}. Sleeping " "for {1} seconds").format(self.response.reason, delay)) time.sleep(delay) else: return None else: return None
def __init__(self, connection, url='', data={}, decode_content=False, \ method=DEFAULT_METHOD, fail_silently=True): """ See the docs above. """ assert method in METHODS self.__connection = connection self.url = url self.data = data self.decode_content = decode_content self.fail_silently = fail_silently headers = { 'Accept': str(self.__connection.content_type), 'Content-type': str(self.__connection.content_type) + \ '; charset=UTF-8' } endpoint_url = self.__connection.base_url + self.url is_xml_request = False if isinstance(self.data, string_types) and self.data[:1] == '<': is_xml_request = True if is_xml_request: headers['Content-type'] = 'text/xml' else: data = json.dumps(data) request_data = { 'url': endpoint_url, 'headers': headers } if DELETE != method: request_data['data'] = data if self.__connection.user and self.__connection.password: request_data['auth'] = HTTPBasicAuth( self.__connection.user, self.__connection.password ) for i in range(connection.retries + 1): self.response = self.call(method, request_data) if self.response.status_code in status_codes.BAD_STATUS_CODES: # If status code is 4** or 5** or raise an exception # means, retry. if i < connection.retries: delay = connection.retry_delay * (2**i) logger.warn( _("Hbase returned error: {0}. Sleeping " "for {1} seconds").format(self.response.reason, delay) ) time.sleep(delay) else: return None else: return None
def _extract_cell_data(cell_data, perfect_dict=PERFECT_DICT): """Extract the cell data. :param list|dict cell_data: :param bool perfect_dict: If set to True, a perfect dictionary is returned (see data structure #2 and #4). Otherwise, ordinary structure is returned (see data structure #1 and #3). If not given, global setting is used. :return list|dict: """ assert isinstance(cell_data, (list, dict)) extracted_cell_data = None # Single column if isinstance(cell_data, dict): extracted_cell_data = Table._extract_column_data( cell_data, perfect_dict=perfect_dict) # Multiple column else: if perfect_dict: extracted_cell_data = {} for column_data in cell_data: d = Table._extract_column_data(column_data, perfect_dict=perfect_dict) for d_key, d_val in d.items(): if PY3: if isinstance(d_val, bytes): d_val = d_val.decode('utf8') if d_key in extracted_cell_data: overlap = set( extracted_cell_data[d_key].keys()) & set( d_val.keys()) if overlap: if 1 == len(overlap): plural = '' keys = overlap.pop() else: plural = 's' keys = '[{}]'.format(','.join( list(overlap))) logger.error( _("Was just about to lose overlapping " "data for key{0} {1}:{2}" "").format(plural, d_key, keys)) else: extracted_cell_data[d_key].update(d_val) else: extracted_cell_data[d_key] = d_val else: extracted_cell_data = {} for column_data in cell_data: extracted_cell_data.update( Table._extract_column_data(column_data, perfect_dict=perfect_dict)) return extracted_cell_data
def _extract_cell_data(cell_data, perfect_dict=PERFECT_DICT): """ Extracts the cell data. :param list|dict cell_data: :param bool perfect_dict: If set to True, a perfect dictionary is returned (see data structure #2 and #4). Otherwise, ordinary structure is returned (see data structure #1 and #3). If not given, global setting is used. :return list|dict: """ assert isinstance(cell_data, (list, dict)) extracted_cell_data = None # Single column if isinstance(cell_data, dict): extracted_cell_data = Table._extract_column_data(cell_data, perfect_dict=perfect_dict) # Multiple column else: if perfect_dict: extracted_cell_data = {} for column_data in cell_data: d = Table._extract_column_data(column_data, perfect_dict=perfect_dict) for d_key, d_val in d.items(): if PY3: if isinstance(d_val, bytes): d_val = d_val.decode('utf8') if d_key in extracted_cell_data: overlap = set(extracted_cell_data[d_key].keys()) & set(d_val.keys()) if overlap: if 1 == len(overlap): plural = '' keys = overlap.pop() else: plural = 's' keys = '[{}]'.format(','.join(list(overlap))) logger.error(_("Was just about to lose overlapping data " "for key{0} {1}:{2}").format(plural, d_key, keys)) else: extracted_cell_data[d_key].update(d_val) else: extracted_cell_data[d_key] = d_val else: extracted_cell_data = {} for column_data in cell_data: extracted_cell_data.update(Table._extract_column_data(column_data, perfect_dict=perfect_dict)) return extracted_cell_data
def _build_delete_url(self, row, column=None, qualifier=None): """ Builds a URL to use for sending the DELETE commands. :param str row: Row id to delete. :param str column: Column :param str qualifier: Column qualifier. :return str: """ if qualifier and not column: raise InvalidArguments(_("Qualifier can't be given without column.")) # Base URL parts = [] parts.append("{table_name}/{row}".format(table_name=self.name, row=row)) if qualifier: parts.append("{0}:{1}".format(column, qualifier)) elif column: parts.append("{0}".format(column)) return '/'.join(parts)
def _build_delete_url(self, row, column=None, qualifier=None): """ Builds a URL to use for sending the DELETE commands. :param str row: Row id to delete. :param str column: Column :param str qualifier: Column qualifier. :return str: """ if qualifier and not column: raise InvalidArguments( _("Qualifier can't be given without column.")) # Base URL parts = [] parts.append("{table_name}/{row}".format(table_name=self.name, row=row)) if qualifier: parts.append("{0}:{1}".format(column, qualifier)) elif column: parts.append("{0}".format(column)) return '/'.join(parts)
def _get(self, row, columns=None, timestamp=None, decode_content=True, number_of_versions=None, raw=False, perfect_dict=None, fail_silently=True): """Retrieve one or more cells from a full row. Retrieve one or more cells from a full row or one or more specified columns in the row, with optional filtering via timestamp, and an optional restriction on the maximum number of versions to return. The `raw` argument is dominant. If given, the raw response it returned. Otherwise, a nice response is returned that does make sense. If `perfect_dict` set to True, then we return a nice dict, instead of a horrible one. In result JSON, the value of the `$` field (key) is the cell data. :param str row: :param list|set|tuple|dict columns: :param timestamp: Not yet used. :param bool decode_content: If set to True, content is decoded using ``stargate.json_decoder.json_decode``. :param int number_of_versions: If provided, multiple versions of the given record are returned. :param bool perfect_dict: :param bool raw: :return dict: """ if self.check_if_exists_on_row_fetch: if not self.exists(fail_silently=fail_silently): return None if perfect_dict is None: perfect_dict = self.connection.perfect_dict # If just one column given as string, make a list of it. if isinstance(columns, string_types): columns = [columns] # Base URL url = "{table_name}/{row}/".format(table_name=self.name, row=row) url += self._build_url_parts(columns) # Handling the timestamps if timestamp: if isinstance(timestamp, str): timestamp = int(timestamp) url += "/{timestamp},{timestamp_plus}/".format( timestamp=str(timestamp), timestamp_plus=str(timestamp + 1)) # If should be versioned, adding additional URL parts. if number_of_versions is not None: assert isinstance(number_of_versions, int) url += '?v={0}'.format(str(number_of_versions)) response = HttpRequest(connection=self.connection, url=url, decode_content=decode_content, fail_silently=fail_silently).get_response() response_content = response.content if raw: return response_content if response_content: try: res = Table._extract_usable_data(response_content, perfect_dict=perfect_dict, with_row_id=False) if isinstance(res, (list, tuple)) and 1 == len(res): return res[0] if not fail_silently: raise ParseError( _("No usable data found in HTTP response.")) except Exception as e: if not fail_silently: raise ParseError( _("Failed to parse the HTTP response. " "Error details: {0}").format(str(e)))
def _get(self, row, columns=None, timestamp=None, decode_content=True, number_of_versions=None, raw=False, perfect_dict=None, fail_silently=True): """Retrieve one or more cells from a full row. Retrieve one or more cells from a full row or one or more specified columns in the row, with optional filtering via timestamp, and an optional restriction on the maximum number of versions to return. The `raw` argument is dominant. If given, the raw response it returned. Otherwise, a nice response is returned that does make sense. If `perfect_dict` set to True, then we return a nice dict, instead of a horrible one. In result JSON, the value of the `$` field (key) is the cell data. :param str row: :param list|set|tuple|dict columns: :param timestamp: Not yet used. :param bool decode_content: If set to True, content is decoded using ``stargate.json_decoder.json_decode``. :param int number_of_versions: If provided, multiple versions of the given record are returned. :param bool perfect_dict: :param bool raw: :return dict: """ if self.check_if_exists_on_row_fetch: if not self.exists(fail_silently=fail_silently): return None if perfect_dict is None: perfect_dict = self.connection.perfect_dict # If just one column given as string, make a list of it. if isinstance(columns, string_types): columns = [columns] # Base URL url = "{table_name}/{row}/".format(table_name=self.name, row=row) url += self._build_url_parts(columns) # Handling the timestamps if timestamp: if isinstance(timestamp, str): timestamp = int(timestamp) url += "/{timestamp},{timestamp_plus}/".format( timestamp = str(timestamp), timestamp_plus = str(timestamp + 1) ) # If should be versioned, adding additional URL parts. if number_of_versions is not None: assert isinstance(number_of_versions, int) url += '?v={0}'.format(str(number_of_versions)) response = HttpRequest( connection = self.connection, url = url, decode_content = decode_content, fail_silently = fail_silently ).get_response() response_content = response.content if raw: return response_content if response_content: try: res = Table._extract_usable_data(response_content, perfect_dict=perfect_dict, with_row_id=False) if isinstance(res, (list, tuple)) and 1 == len(res): return res[0] if not fail_silently: raise ParseError(_("No usable data found in HTTP response.")) except Exception as e: if not fail_silently: raise ParseError(_("Failed to parse the HTTP response. " "Error details: {0}").format(str(e)))