def fetch_for_attribute_given_limit(limit):
response = cubes.cube_single_attribute_elements(
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attribute['id'],
offset=0,
limit=limit)
# Get total number of rows from headers.
total = int(response.headers['x-mstr-total-count'])
# Get attribute elements from the response.
elements = response.json()
# If total number of elements is bigger than the chunk size (limit), fetch them incrementally.
for _offset in range(limit, total, limit):
response = cubes.cube_single_attribute_elements(
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attribute['id'],
offset=_offset,
limit=limit)
elements.extend(response.json())
# Return attribute data.
return {
"attribute_name": attribute['name'],
"attribute_id": attribute['id'],
"elements": elements
}
def __get_attr_elements(self, limit=200000):
"""Get elements of report attributes synchronously. Implements GET /reports/<report_id>/attributes/<attribute_id>/elements"""
attr_elements = []
if self._attributes is not None:
pbar = tqdm(self._attributes, desc="Loading attribute elements", leave=False, disable=(not self.progress_bar))
# Fetch first chunk of attribute elements.
for i, attr in enumerate(pbar):
# Fetch first chunk of attribute elements.
response = cubes.cube_single_attribute_elements(connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr['id'],
offset=self.__OFFSET,
limit=limit)
# Get total number of rows from headers.
total = int(response.headers['x-mstr-total-count'])
# Get attribute elements from the response.
elements = response.json()
# If total number of elements is bigger than the chunk size (limit), fetch them incrementally.
for _offset in range(limit, total, limit):
response = cubes.cube_single_attribute_elements(connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr['id'],
offset=_offset,
limit=limit)
elements.extend(response.json())
# Append attribute data to the list of attributes.
attr_elements.append({"attribute_name": attr['name'],
"attribute_id": attr['id'],
"elements": elements})
pbar.close()
return attr_elements
def __attr_elements(self, limit=25000, progress_bar=True):
"""Get attribute elements. Implements GET /cubes/<cube_id>/attributes/<attribute_id>/elements"""
attr_elements = []
if self._attributes is not None:
with tqdm(total=len(self._attributes),
disable=(not progress_bar)) as fetch_pbar:
if progress_bar:
fetch_pbar.update()
fetch_pbar.set_description("Loading attribute elements")
fetch_pbar.set_postfix(rows=0)
for i_attr, attr in enumerate(self._attributes):
if progress_bar:
fetch_pbar.update()
fetch_pbar.set_description(
"Loading attribute elements")
fetch_pbar.set_postfix(rows=i_attr)
# Fetch first chunk of attribute elements.
res = cubes.cube_single_attribute_elements(
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr['id'],
offset=self.__OFFSET,
limit=limit)
if not res.ok:
msg = "Error retrieving attribute '" + attr[
'name'] + "' elements."
self.__response_handler(response=res, msg=msg)
else:
# Get total number of rows from headers.
total = int(res.headers['x-mstr-total-count'])
# Get attribute elements from the response.
elements = res.json()
# If total number of elements is bigger than the chunk size (limit), fetch them incrementally.
for _offset in range(limit, total, limit):
res = cubes.cube_single_attribute_elements(
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr['id'],
offset=_offset,
limit=limit)
elements.extend(res.json())
# Append attribute data to the list of attributes.
attr_elements.append({
"attribute_name": attr['name'],
"attribute_id": attr['id'],
"elements": elements
})
self._attr_elements = attr_elements
def __attr_elements(self, limit=25000):
"""Get attribute elements. Implements GET /cubes/<cube_id>/attributes/<attribute_id>/elements"""
attr_elements = []
if self._attributes is not None:
for attr in self._attributes:
# Fetch first chunk of attribute elements.
res = cubes.cube_single_attribute_elements(
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr['id'],
offset=self.__OFFSET,
limit=limit)
if not res.ok:
msg = "Error retrieving attribute '" + attr[
'name'] + "' elements."
self.__response_handler(response=res, msg=msg)
else:
# Get total number of rows from headers.
total = int(res.headers['x-mstr-total-count'])
# Get attribute elements from the response.
elements = res.json()
# If total number of elements is bigger than the chunk size (limit), fetch them incrementally.
for _offset in range(limit, total, limit):
res = cubes.cube_single_attribute_elements(
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr['id'],
offset=_offset,
limit=limit)
elements.extend(res.json())
# Append attribute data to the list of attributes.
attr_elements.append({
"attribute_name": attr['name'],
"attribute_id": attr['id'],
"elements": elements
})
self._attr_elements = attr_elements
def __get_attr_elements_async(self, limit=200000):
"""Get attribute elements. Implements GET /cubes/<cube_id>/attributes/<attribute_id>/elements"""
attr_elements = []
if self._attributes is not None:
threads = helper.get_parallel_number(len(self._attributes))
with FuturesSession(executor=ThreadPoolExecutor(max_workers=threads)) as session:
# Fetch first chunk of attribute elements.
futures = self.__fetch_attribute_elements_chunks(session, limit)
pbar = tqdm(futures, desc="Loading attribute elements", leave=False, disable=(not self.progress_bar))
for i, future in enumerate(pbar):
attr = self.attributes[i]
response = future.result()
if not response.ok:
response = cubes.cube_single_attribute_elements(connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr["id"],
offset=0,
limit=limit)
elements = response.json()
# Get total number of rows from headers.
total = int(response.headers['x-mstr-total-count'])
for _offset in range(limit, total, limit):
response = cubes.cube_single_attribute_elements(connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr["id"],
offset=_offset,
limit=limit)
elements.extend(response.json())
# Append attribute data to the list of attributes.
attr_elements.append({"attribute_name": attr['name'],
"attribute_id": attr['id'],
"elements": elements})
pbar.close()
return attr_elements
def generate_cube_single_attr_el_headers(self):
res = cubes.cube_single_attribute_elements(connection=self.connection,
cube_id=self.cube_id,
attribute_id=self.filter_attribute_id)
return res.headers