def publish(self):
"""Publish the uploaded data to the selected dataset.
A dataset can be published just once.
Returns:
response: Response from the Intelligence Server acknowledging the
publication process.
"""
response = datasets.publish(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id)
if not response.ok:
# on error, cancel the previously uploaded data
datasets.publish_cancel(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id)
status = 6 # default initial status
while status != 1:
pub = datasets.publish_status(connection=self._connection, dataset_id=self._dataset_id,
session_id=self._session_id)
pub = pub.json()
status = pub['status']
if status == 1:
print("Dataset '%s' published successfully." % self._name)
def publish(self):
"""Publish the uploaded data to the selected super cube.
A super cube can be published just once.
Returns:
response: Response from the Intelligence Server acknowledging the
publication process.
"""
response = datasets.publish(connection=self._connection,
id=self._id,
session_id=self._session_id)
if not response.ok:
# on error, cancel the previously uploaded data
datasets.publish_cancel(connection=self._connection,
id=self._id,
session_id=self._session_id)
status = 6 # default initial status
while status != 1:
status = self.publish_status()['status']
time.sleep(1)
if status == 1:
print("Super cube '%s' published successfully." % self.name)
def publish(self) -> bool:
"""Publish the uploaded data to the selected super cube.
Note:
A super cube can be published just once.
Returns:
True if the data was published successfully, else False.
"""
response = datasets.publish(connection=self._connection,
id=self._id,
session_id=self._session_id)
if not response.ok:
# on error, cancel the previously uploaded data
datasets.publish_cancel(connection=self._connection,
id=self._id,
session_id=self._session_id)
return False
status = 6 # default initial status
while status != 1:
status = self.publish_status()['status']
time.sleep(1)
if status == 1:
# clear instance_id to force new instance creation
self.instance_id = None
if config.verbose:
logger.info(
f"Super cube '{self.name}' published successfully.")
return True
def update(self, chunksize=100000):
"""Updates a dataset with new data.
Args:
chunksize (int, optional): Number of rows to transmit to the server with each request.
"""
# form request body and create a session for data uploads
self.__form_upload_body()
response = datasets.upload_session(connection=self._connection,
dataset_id=self._dataset_id, body=self.__upload_body)
response_json = response.json()
self._session_id = response_json['uploadSessionId']
# upload each table
for ix, _table in enumerate(self._tables):
_df, _name = _table["data_frame"], _table["table_name"]
# break the data up into chunks using a generator
chunks = (_df[i:i + chunksize] for i in range(0, _df.shape[0], chunksize))
total = _df.shape[0]
# Count the number of iterations
it_total = int(total/chunksize) + (total % chunksize != 0)
pbar = tqdm(chunks, total=it_total, disable=(not self.progress_bar))
for index, chunk in enumerate(pbar):
pbar.set_description("Uploading {}/{}".format(ix+1, len(self._tables)))
# base64 encode the data
encoder = Encoder(data_frame=chunk, dataset_type='multi')
b64_enc = encoder.encode
# form body of the request
body = {"tableName": _name,
"index": index + 1,
"data": b64_enc}
# make request to upload the data
response = datasets.upload(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id,
body=body)
if not response.ok:
# on error, cancel the previously uploaded data
datasets.publish_cancel(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id)
pbar.set_postfix(rows=min((index+1)*chunksize, total))
pbar.close()
self._tables = []
def publish(self):
"""Publish the uploaded data to the selected dataset.
Returns:
response: Response from the Intelligence Server acknowledging the publication process.
"""
response = datasets.publish(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id)
if not response.ok:
# on error, cancel the previously uploaded data
self.__response_handler(
response=response,
msg="Error publishing uploaded data. Cancelling publication.")
datasets.publish_cancel(connection=self._connection,
dataset_id=self._dataset_id,
session_id=self._session_id)
return response
def update(self, chunksize: int = 100000, auto_publish: bool = True):
"""Updates a super cube with new data.
Args:
chunksize (int, optional): Number of rows to transmit to the server
with each request.
auto_publish: If True, automatically publishes the data used to
update the super cube definition to the super cube. If False,
simply updates the super cube but does not publish it.
"""
# form request body and create a session for data uploads
self.__form_upload_body()
response = datasets.upload_session(connection=self._connection,
id=self._id,
body=self.__upload_body)
response_json = response.json()
self._session_id = response_json['uploadSessionId']
# upload each table
for ix, _table in enumerate(self._tables):
_df, _name = _table["data_frame"], _table["table_name"]
# break the data up into chunks using a generator
chunks = (_df[i:i + chunksize]
for i in range(0, _df.shape[0], chunksize))
total = _df.shape[0]
# Count the number of iterations
it_total = int(total / chunksize) + (total % chunksize != 0)
pbar = tqdm(chunks,
total=it_total,
disable=(not self._progress_bar))
for index, chunk in enumerate(pbar):
pbar.set_description(f"Uploading {ix + 1}/{len(self._tables)}")
# base64 encode the data
encoder = Encoder(data_frame=chunk, dataset_type='multi')
b64_enc = encoder.encode
# form body of the request
body = {
"tableName": _name,
"index": index + 1,
"data": b64_enc,
}
# make request to upload the data
response = datasets.upload(
connection=self._connection,
id=self._id,
session_id=self._session_id,
body=body,
)
if not response.ok:
# on error, cancel the previously uploaded data
datasets.publish_cancel(connection=self._connection,
id=self._id,
session_id=self._session_id)
pbar.set_postfix(rows=min((index + 1) * chunksize, total))
pbar.close()
self._tables = []
# if desired, automatically publish the data to the new super cube
if auto_publish:
self.publish()
