def __init__(self, host=HOST, port=PORT, url=None, user=USER, password=PASSWORD, secure=False, \
verify_ssl=True, content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT,
retries=RETRIES, retry_delay=RETRY_DELAY):
"""
Creates a new connection instance.
See docs above.
"""
if not content_type in CONTENT_TYPES:
raise ImproperlyConfigured(
_("Invalid ``content_type`` {0} value.".format(content_type)))
if not host:
raise ImproperlyConfigured(
_("Invalid ``host`` {0} value.".format(host)))
if not port:
raise ImproperlyConfigured(
_("Invalid ``port`` {0} value.".format(port)))
self.host = host
self.port = port
self.user = user
self.password = password
self.secure = secure
self.verify_ssl = verify_ssl
self.content_type = CONTENT_TYPES_DICT[content_type]
self.perfect_dict = perfect_dict
self.retries = retries
self.retry_delay = retry_delay
self.__connect(url)
def __init__(self, host=HOST, port=PORT, url=None, user=USER, password=PASSWORD, secure=False, \
content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT,
retries=RETRIES, retry_delay=RETRY_DELAY):
"""
Creates a new connection instance.
See docs above.
"""
if not content_type in CONTENT_TYPES:
raise ImproperlyConfigured(_("Invalid ``content_type`` {0} value.".format(content_type)))
if not host:
raise ImproperlyConfigured(_("Invalid ``host`` {0} value.".format(host)))
if not port:
raise ImproperlyConfigured(_("Invalid ``port`` {0} value.".format(port)))
self.host = host
self.port = port
self.user = user
self.password = password
self.secure = secure
self.content_type = CONTENT_TYPES_DICT[content_type]
self.perfect_dict = perfect_dict
self.retries = retries
self.retry_delay = retry_delay
self.__connect(url)
def __init__(self, host=HOST, port=PORT, user=USER, password=PASSWORD, secure=False, \
content_type=DEFAULT_CONTENT_TYPE, perfect_dict=PERFECT_DICT):
"""
Creates a new connection instance.
See docs above.
"""
if not content_type in CONTENT_TYPES:
raise ImproperlyConfigured(
_("Invalid ``content_type`` {0} value.".format(content_type)))
if not host:
raise ImproperlyConfigured(
_("Invalid ``host`` {0} value.".format(host)))
if not port:
raise ImproperlyConfigured(
_("Invalid ``port`` {0} value.".format(port)))
self.host = host
self.port = port
self.user = user
self.password = password
self.secure = secure
self.content_type = CONTENT_TYPES_DICT[content_type]
self.perfect_dict = perfect_dict
self.__connect()
def fetch_all_rows(self, with_row_id=False, raw=False, perfect_dict=None,
flat=False, filter_string=None, scanner_config={},
fail_silently=True):
"""Fetch all table rows.
:param bool with_row_id: If set to True, returned along with row id.
:param bool raw: If set to True, raw response is returned.
:param bool perfect_dict: If set to True, a perfect dict structure is
used for output data.
:param bool flat:
:param string filter_string: If set, applies the given filter string
to the scanner.
:param dict scanner_config:
:param mixed flat_silently.
:return list:
:example:
>>> filter_string = '{"type": "RowFilter", "op": "EQUAL", "comparator": '
>>> '{"type": "RegexStringComparator", "value": "^row_1.+"}}'
>>> rows = self.table.fetch_all_rows(
>>> with_row_id = True,
>>> perfect_dict = perfect_dict,
>>> filter_string = row_filter_string
>>> )
"""
if self.check_if_exists_on_scanner_operations:
if not self.exists(fail_silently=fail_silently):
return None
if perfect_dict is None:
perfect_dict = self.connection.perfect_dict
try:
scanner = self._scanner(filter_string=filter_string,
data=scanner_config,
fail_silently=fail_silently)
except HTTPError as e:
if fail_silently:
return []
raise DoesNotExist(_("""Table "{0}" does not exist."""
"".format(self.name)))
if scanner:
res = scanner.results(perfect_dict=perfect_dict,
with_row_id=with_row_id,
raw=raw)
scanner.delete ()
else:
if fail_silently:
return None
raise DoesNotExist(_("""Table "{0}" does not exist.""".format(self.name)))
if flat:
res = list(res)
return res
def fetch_all_rows(self, with_row_id=False, raw=False, perfect_dict=None, flat=False, filter_string=None, \
scanner_config='', fail_silently=True):
"""
Fetches all table rows.
:param bool with_row_id: If set to True, returned along with row id.
:param bool raw: If set to True, raw response is returned.
:param bool perfect_dict: If set to True, a perfect dict struture is used for output data.
:param string filter_string: If set, applies the given filter string to the scanner.
:returns list:
:example:
>>> filter_string = '{"type": "RowFilter", "op": "EQUAL", "comparator": '
>>> '{"type": "RegexStringComparator", "value": "^row_1.+"}}'
>>> rows = self.table.fetch_all_rows(
>>> with_row_id = True,
>>> perfect_dict = perfect_dict,
>>> filter_string = row_filter_string
>>> )
"""
if self.check_if_exists_on_scanner_operations:
if not self.exists(fail_silently=fail_silently):
return None
if perfect_dict is None:
perfect_dict = self.connection.perfect_dict
try:
scanner = self._scanner(filter_string=filter_string,
data=scanner_config,
fail_silently=fail_silently)
except HTTPError as e:
if fail_silently:
return []
raise DoesNotExist(
_("""Table "{0}" does not exist.""".format(self.name)))
if scanner:
res = scanner.results(perfect_dict=perfect_dict,
with_row_id=with_row_id,
raw=raw)
scanner.delete()
else:
if fail_silently:
return None
raise DoesNotExist(
_("""Table "{0}" does not exist.""".format(self.name)))
if flat:
res = list(res)
return res
def __init__(self, connection, url='', data={}, decode_content=False, \
method=DEFAULT_METHOD, fail_silently=True):
"""
See the docs above.
"""
assert method in METHODS
self.__connection = connection
self.url = url
self.data = data
self.decode_content = decode_content
self.fail_silently = fail_silently
self.verify_ssl = connection.verify_ssl
if not self.verify_ssl:
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
headers = {
'Accept': str(self.__connection.content_type),
'Content-type': str(self.__connection.content_type) + \
'; charset=UTF-8'
}
endpoint_url = self.__connection.base_url + self.url
is_xml_request = False
if isinstance(self.data, string_types) and self.data[:1] == '<':
is_xml_request = True
if is_xml_request:
headers['Content-type'] = 'text/xml'
else:
data = json.dumps(data)
request_data = {'url': endpoint_url, 'headers': headers}
if DELETE != method:
request_data['data'] = data
if self.__connection.user and self.__connection.password:
request_data['auth'] = HTTPBasicAuth(self.__connection.user,
self.__connection.password)
for i in range(connection.retries + 1):
self.response = self.call(method, request_data)
if self.response.status_code in status_codes.BAD_STATUS_CODES:
# If status code is 4** or 5** or raise an exception
# means, retry.
if i < connection.retries:
delay = connection.retry_delay * (2**i)
logger.warn(
_("Hbase returned error: {0}. Sleeping "
"for {1} seconds").format(self.response.reason,
delay))
time.sleep(delay)
else:
return None
else:
return None
def __init__(self, connection, url='', data={}, decode_content=False, \
method=DEFAULT_METHOD, fail_silently=True):
"""
See the docs above.
"""
assert method in METHODS
self.__connection = connection
self.url = url
self.data = data
self.decode_content = decode_content
self.fail_silently = fail_silently
headers = {
'Accept': str(self.__connection.content_type),
'Content-type': str(self.__connection.content_type) + \
'; charset=UTF-8'
}
endpoint_url = self.__connection.base_url + self.url
is_xml_request = False
if isinstance(self.data, string_types) and self.data[:1] == '<':
is_xml_request = True
if is_xml_request:
headers['Content-type'] = 'text/xml'
else:
data = json.dumps(data)
request_data = {
'url': endpoint_url,
'headers': headers
}
if DELETE != method:
request_data['data'] = data
if self.__connection.user and self.__connection.password:
request_data['auth'] = HTTPBasicAuth(
self.__connection.user,
self.__connection.password
)
for i in range(connection.retries + 1):
self.response = self.call(method, request_data)
if self.response.status_code in status_codes.BAD_STATUS_CODES:
# If status code is 4** or 5** or raise an exception
# means, retry.
if i < connection.retries:
delay = connection.retry_delay * (2**i)
logger.warn(
_("Hbase returned error: {0}. Sleeping "
"for {1} seconds").format(self.response.reason, delay)
)
time.sleep(delay)
else:
return None
else:
return None
def _extract_cell_data(cell_data, perfect_dict=PERFECT_DICT):
"""Extract the cell data.
:param list|dict cell_data:
:param bool perfect_dict: If set to True, a perfect dictionary is
returned (see data structure #2 and #4). Otherwise, ordinary
structure is returned (see data structure #1 and #3). If not given,
global setting is used.
:return list|dict:
"""
assert isinstance(cell_data, (list, dict))
extracted_cell_data = None
# Single column
if isinstance(cell_data, dict):
extracted_cell_data = Table._extract_column_data(
cell_data, perfect_dict=perfect_dict)
# Multiple column
else:
if perfect_dict:
extracted_cell_data = {}
for column_data in cell_data:
d = Table._extract_column_data(column_data,
perfect_dict=perfect_dict)
for d_key, d_val in d.items():
if PY3:
if isinstance(d_val, bytes):
d_val = d_val.decode('utf8')
if d_key in extracted_cell_data:
overlap = set(
extracted_cell_data[d_key].keys()) & set(
d_val.keys())
if overlap:
if 1 == len(overlap):
plural = ''
keys = overlap.pop()
else:
plural = 's'
keys = '[{}]'.format(','.join(
list(overlap)))
logger.error(
_("Was just about to lose overlapping "
"data for key{0} {1}:{2}"
"").format(plural, d_key, keys))
else:
extracted_cell_data[d_key].update(d_val)
else:
extracted_cell_data[d_key] = d_val
else:
extracted_cell_data = {}
for column_data in cell_data:
extracted_cell_data.update(
Table._extract_column_data(column_data,
perfect_dict=perfect_dict))
return extracted_cell_data
def _extract_cell_data(cell_data, perfect_dict=PERFECT_DICT):
"""
Extracts the cell data.
:param list|dict cell_data:
:param bool perfect_dict: If set to True, a perfect dictionary is returned (see data structure #2 and #4).
Otherwise, ordinary structure is returned (see data structure #1 and #3). If not given, global setting
is used.
:return list|dict:
"""
assert isinstance(cell_data, (list, dict))
extracted_cell_data = None
# Single column
if isinstance(cell_data, dict):
extracted_cell_data = Table._extract_column_data(cell_data, perfect_dict=perfect_dict)
# Multiple column
else:
if perfect_dict:
extracted_cell_data = {}
for column_data in cell_data:
d = Table._extract_column_data(column_data, perfect_dict=perfect_dict)
for d_key, d_val in d.items():
if PY3:
if isinstance(d_val, bytes):
d_val = d_val.decode('utf8')
if d_key in extracted_cell_data:
overlap = set(extracted_cell_data[d_key].keys()) & set(d_val.keys())
if overlap:
if 1 == len(overlap):
plural = ''
keys = overlap.pop()
else:
plural = 's'
keys = '[{}]'.format(','.join(list(overlap)))
logger.error(_("Was just about to lose overlapping data "
"for key{0} {1}:{2}").format(plural, d_key, keys))
else:
extracted_cell_data[d_key].update(d_val)
else:
extracted_cell_data[d_key] = d_val
else:
extracted_cell_data = {}
for column_data in cell_data:
extracted_cell_data.update(Table._extract_column_data(column_data, perfect_dict=perfect_dict))
return extracted_cell_data
def _build_delete_url(self, row, column=None, qualifier=None):
"""
Builds a URL to use for sending the DELETE commands.
:param str row: Row id to delete.
:param str column: Column
:param str qualifier: Column qualifier.
:return str:
"""
if qualifier and not column:
raise InvalidArguments(_("Qualifier can't be given without column."))
# Base URL
parts = []
parts.append("{table_name}/{row}".format(table_name=self.name, row=row))
if qualifier:
parts.append("{0}:{1}".format(column, qualifier))
elif column:
parts.append("{0}".format(column))
return '/'.join(parts)
def _build_delete_url(self, row, column=None, qualifier=None):
"""
Builds a URL to use for sending the DELETE commands.
:param str row: Row id to delete.
:param str column: Column
:param str qualifier: Column qualifier.
:return str:
"""
if qualifier and not column:
raise InvalidArguments(
_("Qualifier can't be given without column."))
# Base URL
parts = []
parts.append("{table_name}/{row}".format(table_name=self.name,
row=row))
if qualifier:
parts.append("{0}:{1}".format(column, qualifier))
elif column:
parts.append("{0}".format(column))
return '/'.join(parts)
def _get(self,
row,
columns=None,
timestamp=None,
decode_content=True,
number_of_versions=None,
raw=False,
perfect_dict=None,
fail_silently=True):
"""Retrieve one or more cells from a full row.
Retrieve one or more cells from a full row or one or more specified
columns in the row, with optional filtering via timestamp, and an
optional restriction on the maximum number of versions to return.
The `raw` argument is dominant. If given, the raw response it returned.
Otherwise, a nice response is returned that does make sense.
If `perfect_dict` set to True, then we return a nice dict, instead of a
horrible one.
In result JSON, the value of the `$` field (key) is the cell data.
:param str row:
:param list|set|tuple|dict columns:
:param timestamp: Not yet used.
:param bool decode_content: If set to True, content is decoded using
``stargate.json_decoder.json_decode``.
:param int number_of_versions: If provided, multiple versions of the
given record are returned.
:param bool perfect_dict:
:param bool raw:
:return dict:
"""
if self.check_if_exists_on_row_fetch:
if not self.exists(fail_silently=fail_silently):
return None
if perfect_dict is None:
perfect_dict = self.connection.perfect_dict
# If just one column given as string, make a list of it.
if isinstance(columns, string_types):
columns = [columns]
# Base URL
url = "{table_name}/{row}/".format(table_name=self.name, row=row)
url += self._build_url_parts(columns)
# Handling the timestamps
if timestamp:
if isinstance(timestamp, str):
timestamp = int(timestamp)
url += "/{timestamp},{timestamp_plus}/".format(
timestamp=str(timestamp), timestamp_plus=str(timestamp + 1))
# If should be versioned, adding additional URL parts.
if number_of_versions is not None:
assert isinstance(number_of_versions, int)
url += '?v={0}'.format(str(number_of_versions))
response = HttpRequest(connection=self.connection,
url=url,
decode_content=decode_content,
fail_silently=fail_silently).get_response()
response_content = response.content
if raw:
return response_content
if response_content:
try:
res = Table._extract_usable_data(response_content,
perfect_dict=perfect_dict,
with_row_id=False)
if isinstance(res, (list, tuple)) and 1 == len(res):
return res[0]
if not fail_silently:
raise ParseError(
_("No usable data found in HTTP response."))
except Exception as e:
if not fail_silently:
raise ParseError(
_("Failed to parse the HTTP response. "
"Error details: {0}").format(str(e)))
def _get(self, row, columns=None, timestamp=None, decode_content=True,
number_of_versions=None, raw=False, perfect_dict=None,
fail_silently=True):
"""Retrieve one or more cells from a full row.
Retrieve one or more cells from a full row or one or more specified
columns in the row, with optional filtering via timestamp, and an
optional restriction on the maximum number of versions to return.
The `raw` argument is dominant. If given, the raw response it returned.
Otherwise, a nice response is returned that does make sense.
If `perfect_dict` set to True, then we return a nice dict, instead of a
horrible one.
In result JSON, the value of the `$` field (key) is the cell data.
:param str row:
:param list|set|tuple|dict columns:
:param timestamp: Not yet used.
:param bool decode_content: If set to True, content is decoded using
``stargate.json_decoder.json_decode``.
:param int number_of_versions: If provided, multiple versions of the
given record are returned.
:param bool perfect_dict:
:param bool raw:
:return dict:
"""
if self.check_if_exists_on_row_fetch:
if not self.exists(fail_silently=fail_silently):
return None
if perfect_dict is None:
perfect_dict = self.connection.perfect_dict
# If just one column given as string, make a list of it.
if isinstance(columns, string_types):
columns = [columns]
# Base URL
url = "{table_name}/{row}/".format(table_name=self.name, row=row)
url += self._build_url_parts(columns)
# Handling the timestamps
if timestamp:
if isinstance(timestamp, str):
timestamp = int(timestamp)
url += "/{timestamp},{timestamp_plus}/".format(
timestamp = str(timestamp),
timestamp_plus = str(timestamp + 1)
)
# If should be versioned, adding additional URL parts.
if number_of_versions is not None:
assert isinstance(number_of_versions, int)
url += '?v={0}'.format(str(number_of_versions))
response = HttpRequest(
connection = self.connection,
url = url,
decode_content = decode_content,
fail_silently = fail_silently
).get_response()
response_content = response.content
if raw:
return response_content
if response_content:
try:
res = Table._extract_usable_data(response_content,
perfect_dict=perfect_dict,
with_row_id=False)
if isinstance(res, (list, tuple)) and 1 == len(res):
return res[0]
if not fail_silently:
raise ParseError(_("No usable data found in HTTP response."))
except Exception as e:
if not fail_silently:
raise ParseError(_("Failed to parse the HTTP response. "
"Error details: {0}").format(str(e)))
