def publish(self):
"""Publish the uploaded data to the selected dataset. A dataset can be published just once.
Returns:
response: Response from the Intelligence Server acknowledging the publication process.
"""
response = datasets.publish(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id,
verbose=helper.debug())
if not response.ok:
# on error, cancel the previously uploaded data
datasets.publish_cancel(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id,
verbose=helper.debug())
status = 6 # default initial status
while status != 1:
pub = datasets.publish_status(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id)
pub = pub.json()
status = pub['status']
if status == 1:
print("Dataset '%s' published successfully." % self._name)
def __definition(self):
"""Get the definition of a cube, including attributes and metrics. Implements GET /v2/cubes/<cube_id>."""
res = cubes.cube_definition(connection=self._connection,
cube_id=self._cube_id,
verbose=helper.debug())
_definition = res.json()
full_attributes = _definition["definition"]["availableObjects"][
"attributes"]
full_metrics = _definition["definition"]["availableObjects"]["metrics"]
self._attributes = [{
'name': attr['name'],
'id': attr['id']
} for attr in full_attributes]
self._metrics = [{
'name': metr['name'],
'id': metr['id']
} for metr in full_metrics]
self._tables = self.__multitable_definition().keys()
row_counts = [
'Row Count - {}'.format(table_name) for table_name in self._tables
]
self._row_counts = list(
filter(lambda x: x['name'] in row_counts, self.metrics))
def __get_chunk(self, instance_id, offset, limit):
return reports.report_instance_id(connection=self._connection,
report_id=self._report_id,
instance_id=instance_id,
offset=offset,
limit=limit,
verbose=helper.debug())
def __get_chunk(self, instance_id, offset, limit):
return cubes.cube_instance_id(connection=self._connection,
cube_id=self._cube_id,
instance_id=instance_id,
offset=offset,
limit=limit,
verbose=helper.debug())
def __get_attr_elements(self, limit=200000):
"""Get elements of report attributes synchronously.
Implements GET /reports/<report_id>/attributes/<attribute_id>/elements
"""
attr_elements = []
if self.attributes:
pbar = tqdm(self.attributes,
desc="Loading attribute elements",
leave=False,
disable=(not self.progress_bar))
# Fetch first chunk of attribute elements.
for i, attr in enumerate(pbar):
# Fetch first chunk of attribute elements.
response = cubes.cube_single_attribute_elements(
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr['id'],
offset=0,
limit=limit,
verbose=helper.debug())
# Get total number of rows from headers.
total = int(response.headers['x-mstr-total-count'])
# Get attribute elements from the response.
elements = response.json()
# If total number of elements is bigger than the chunk size (limit), fetch them incrementally.
for _offset in range(limit, total, limit):
response = cubes.cube_single_attribute_elements(
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attr['id'],
offset=_offset,
limit=limit,
verbose=helper.debug())
elements.extend(response.json())
# Append attribute data to the list of attributes.
attr_elements.append({
"attribute_name": attr['name'],
"attribute_id": attr['id'],
"elements": elements
})
pbar.close()
return attr_elements
def close(self):
"""Closes a connection with MicroStrategy REST API"""
authentication.logout(connection=self, verbose=helper.debug())
if self.verbose:
print(
"Connection to MicroStrategy Intelligence Server has been closed"
)
def __load_definition(self):
"""Load definition of an existing dataset."""
response = datasets.dataset_definition(connection=self._connection,
dataset_id=self._dataset_id,
verbose=helper.debug())
self._definition = response.json()
self._name = self._definition['name']
def renew(self):
"""Checks if the session is still alive. If so, renews the session and extends session expiration."""
status = authentication.session_renew(connection=self,
verbose=helper.debug())
if status.status_code == 204:
if self.verbose:
print(
"Your connection to MicroStrategy Intelligence Server was renewed."
)
else:
response = authentication.login(connection=self,
verbose=helper.debug())
self.session.headers['X-MSTR-AuthToken'] = response.headers[
'X-MSTR-AuthToken']
if self.verbose:
print(
"""Connection with MicroStrategy Intelligence Server was not active.
\rNew connection has been established.""")
def delete(self):
"""Delete a dataset that was previously created using the REST API.
"""
datasets.delete_dataset(connection=self._connection,
dataset_id=self._dataset_id,
verbose=helper.debug())
if self.verbose:
print("Successfully deleted dataset ID: '{}'.".format(
self._dataset_id))
def connect(self):
"""Authenticates the user and creates a new connection with the Intelligence Server. If an active connection is
detected, the session is renewed"""
response = authentication.session_renew(connection=self,
verbose=helper.debug())
if not response.ok:
response = authentication.login(connection=self,
verbose=helper.debug())
self.session.headers['X-MSTR-AuthToken'] = response.headers[
'X-MSTR-AuthToken']
if self.verbose:
print(
"Connection to MicroStrategy Intelligence Server has been established."
)
else:
if self.verbose:
print(
"Connection to MicroStrategy Intelligence Server was renewed."
)
def status(self):
"""Checks if the session is still alive."""
status = authentication.session_status(connection=self,
verbose=helper.debug())
if status.status_code == 200:
print("Connection to MicroStrategy Intelligence Server is active.")
else:
print(
"Connection to MicroStrategy Intelligence Server is not active."
)
def __fetch_attribute_elements_chunks(self, future_session, limit):
# Fetch add'l rows from this object instance from the intelligence server
return [
cubes.cube_single_attribute_elements_coroutine(
future_session,
connection=self._connection,
cube_id=self._cube_id,
attribute_id=attribute['id'],
offset=0,
limit=limit,
verbose=helper.debug()) for attribute in self.attributes
]
def publish_status(self):
"""Check the status of data that was uploaded to a dataset.
Returns:
status: The status of the publication process as a dictionary. In the 'status' key, "1" denotes completion.
"""
response = datasets.publish_status(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id,
verbose=helper.debug())
status = response.json()
return status
def certify(self):
"""Certify the uploaded dataset.
Returns:
response: Response from the Intelligence Server acknowledging the certification process.
"""
response = datasets.toggle_certification(connection=self._connection,
dataset_id=self._dataset_id,
verbose=helper.debug())
if self.verbose:
print("The dataset with ID: '{}' has been certified.".format(
self._dataset_id))
else:
return response
def __fetch_chunks_future(self, future_session, pagination, instance_id,
limit):
# Fetch add'l rows from this object instance from the intelligence server
return [
cubes.cube_instance_id_coroutine(future_session,
connection=self._connection,
cube_id=self._cube_id,
instance_id=instance_id,
offset=_offset,
limit=limit,
verbose=helper.debug()) for
_offset in range(self._initial_limit, pagination['total'], limit)
]
def __definition(self):
"""Get the definition of a report, including attributes and metrics. Implements GET /v2/reports/<report_id>"""
response = reports.report_definition(connection=self._connection,
report_id=self._report_id,
verbose=helper.debug()).json()
grid = response["definition"]["grid"]
available_objects = response['definition']['availableObjects']
if parse_version(self._connection.iserver_version) >= parse_version(
"11.2.0100"):
self._subtotals = grid["subtotals"]
self._name = response["name"]
self.cross_tab = grid["crossTab"]
# Check if report have custom groups or consolidations
if available_objects['customGroups']:
helper.exception_handler(
msg="Reports with custom groups are not supported.",
exception_type=ImportError)
if available_objects['consolidations']:
helper.exception_handler(
msg="Reports with consolidations are not supported.",
exception_type=ImportError)
full_attributes = []
for row in grid["rows"]:
if row["type"] == "attribute":
full_attributes.append(row)
for column in grid["columns"]:
if column["type"] == "attribute":
full_attributes.append(column)
self._attributes = [{
'name': attr['name'],
'id': attr['id']
} for attr in full_attributes]
# Retrieve metrics from the report grid (metrics selected only in the report)
metrics_position = grid.get("metricsPosition")
if metrics_position is None:
self._metrics = []
else:
full_metrics = grid[metrics_position["axis"]][
metrics_position["index"]]["elements"]
self._metrics = [{
'name': metr['name'],
'id': metr['id']
} for metr in full_metrics]
def __info(self):
"""Get metadata for specific cubes. Implements GET /cubes to retrieve basic metadata."""
res = cubes.cube_info(connection=self._connection,
cube_id=self._cube_id,
verbose=helper.debug())
_info = res.json()["cubesInfos"][0]
self._name = _info["cubeName"]
self._owner_id = _info["ownerId"]
self._path = _info["path"]
self._last_modified = _info["modificationTime"]
self._server_mode = _info["serverMode"]
self._size = _info["size"]
self._status = _info["status"]
def __initialize_cube(self, limit):
inst_pbar = tqdm(
desc='Initializing an instance of a cube. Please wait...',
bar_format='{desc}',
leave=False,
ncols=280,
disable=(not self.progress_bar))
# Request a new instance, set instance id
response = cubes.cube_instance(connection=self._connection,
cube_id=self._cube_id,
body=self.__filter._filter_body(),
offset=0,
limit=self._initial_limit,
verbose=helper.debug())
inst_pbar.close()
return response
def upload_status(self, connection, dataset_id, session_id):
"""Check the status of data that was uploaded to a dataset.
Args:
connection: MicroStrategy connection object returned by `connection.Connection()`.
dataset_id (str): Identifier of a pre-existing dataset.
session_id (str): Identifer of the server session used for collecting uploaded data.
"""
response = datasets.publish_status(connection=connection,
dataset_id=dataset_id,
session_id=session_id,
verbose=helper.debug())
helper.response_handler(
response=response,
msg="Publication status for dataset with ID: '{}':".format(
dataset_id),
throw_error=False)
def __get_attr_elements_async(self, limit=200000):
"""Get elements of report attributes asynchronously.
Implements GET /reports/<report_id>/attributes/<attribute_id>/elements
"""
attr_elements = []
if self.attributes:
threads = helper.get_parallel_number(len(self.attributes))
with FuturesSession(
executor=ThreadPoolExecutor(max_workers=threads),
session=self._connection.session) as session:
# Fetch first chunk of attribute elements.
futures = self.__fetch_attribute_elements_chunks(
session, limit)
pbar = tqdm(futures,
desc="Loading attribute elements",
leave=False,
disable=(not self.progress_bar))
for i, future in enumerate(pbar):
attr = self.attributes[i]
response = future.result()
if not response.ok:
helper.response_handler(
response, "Error getting attribute " +
attr['name'] + " elements")
elements = response.json()
# Get total number of rows from headers.
total = int(response.headers['x-mstr-total-count'])
for _offset in range(limit, total, limit):
response = reports.report_single_attribute_elements(
connection=self._connection,
report_id=self._report_id,
attribute_id=attr["id"],
offset=_offset,
limit=limit,
verbose=helper.debug())
elements.extend(response.json())
# Append attribute data to the list of attributes.
attr_elements.append({
"attribute_name": attr['name'],
"attribute_id": attr['id'],
"elements": elements
})
pbar.close()
return attr_elements
def create(self,
folder_id=None,
auto_upload=True,
auto_publish=True,
chunksize=100000):
"""Creates a new dataset.
Args:
folder_id (str, optional): ID of the shared folder that the dataset should be created within. If `None`,
defaults to the user's My Reports folder.
auto_upload (bool, optional): If True, automatically uploads the data to the I-Server. If False, simply
creates the dataset definition but does not upload data to it.
auto_publish (bool, optional): If True, automatically publishes the data used to create the dataset
definition. If False, simply creates the dataset but does not publish it. To publish the dataset, data
has to be uploaded first.
chunksize (int, optional): Number of rows to transmit to the I-Server with each request when uploading.
"""
if auto_publish and not auto_upload:
helper.exception_handler(
"Data needs to be uploaded to the I-Server before the dataset can be published.",
ValueError)
if folder_id is not None:
self._folder_id = folder_id
else:
self._folder_id = ""
# generate model of the dataset
self.__build_model()
# makes request to create the dataset
response = datasets.create_multitable_dataset(
connection=self._connection,
body=self.__model,
verbose=helper.debug())
response_json = response.json()
self._dataset_id = response_json['id']
if self.verbose:
print("Created dataset '{}' with ID: '{}'.".format(
*[self._name, self._dataset_id]))
if auto_upload:
self.update(chunksize=chunksize, auto_publish=auto_publish)
def __initialize_report(self, limit):
inst_pbar = tqdm(
desc='Initializing an instance of a report. Please wait...',
bar_format='{desc}',
leave=False,
ncols=285,
disable=(not self.progress_bar))
# Switch off subtotals if I-Server version is higher than 11.2.1
body = self.__filter._filter_body()
if parse_version(self._connection.iserver_version) >= parse_version(
"11.2.0100"):
self._subtotals["visible"] = False
body["subtotals"] = {"visible": self._subtotals["visible"]}
# Request a new instance, set instance id
response = reports.report_instance(connection=self._connection,
report_id=self._report_id,
body=body,
offset=0,
limit=self._initial_limit,
verbose=helper.debug())
inst_pbar.close()
return response
def __select_project(self):
# Fetch the list of projects the user has access to
msg = "Error connecting to project '{}'. Check project name and try again.".format(
self.project_name)
response = projects.projects(connection=self,
error_msg=msg,
verbose=helper.debug())
# Find which project ID matches the project name provided
_projects = response.json()
if self.project_name is not None:
# Find which project ID matches the project name provided
for _project in _projects:
if _project[
'name'] == self.project_name: # Enter the desired project name here
self.project_id = _project['id']
else:
# Find which project name matches the project ID provided
for _project in _projects:
if _project[
'id'] == self.project_id: # Enter the desired project id here
self.project_name = _project['name']
self.session.headers['X-MSTR-ProjectID'] = self.project_id
def update(self, chunksize=100000, auto_publish=True):
"""Updates a dataset with new data.
Args:
chunksize (int, optional): Number of rows to transmit to the server with each request.
auto_publish: If True, automatically publishes the data used to update the dataset definition to the dataset.
If False, simply updates the dataset but does not publish it.
"""
# form request body and create a session for data uploads
self.__form_upload_body()
response = datasets.upload_session(connection=self._connection,
dataset_id=self._dataset_id,
body=self.__upload_body,
verbose=helper.debug())
response_json = response.json()
self._session_id = response_json['uploadSessionId']
# upload each table
for ix, _table in enumerate(self._tables):
_df, _name = _table["data_frame"], _table["table_name"]
# break the data up into chunks using a generator
chunks = (_df[i:i + chunksize]
for i in range(0, _df.shape[0], chunksize))
total = _df.shape[0]
# Count the number of iterations
it_total = int(total / chunksize) + (total % chunksize != 0)
pbar = tqdm(chunks,
total=it_total,
disable=(not self.progress_bar))
for index, chunk in enumerate(pbar):
pbar.set_description("Uploading {}/{}".format(
ix + 1, len(self._tables)))
# base64 encode the data
encoder = Encoder(data_frame=chunk, dataset_type='multi')
b64_enc = encoder.encode
# form body of the request
body = {
"tableName": _name,
"index": index + 1,
"data": b64_enc
}
# make request to upload the data
response = datasets.upload(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id,
body=body,
verbose=helper.debug())
if not response.ok:
# on error, cancel the previously uploaded data
datasets.publish_cancel(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id,
verbose=helper.debug())
pbar.set_postfix(rows=min((index + 1) * chunksize, total))
pbar.close()
self._tables = []
# if desired, automatically publish the data to the new dataset
if auto_publish:
self.publish()
