class WaveDataUploader:
"""
This class is used to create a wave dataset uploader object by given config file location.
"""
def __init__(self, configFile, mode):
"""
Constructor for wave data uploader.
:param configFile: config file name
"""
self._uploadMode = mode
self._config = ConfigParser()
self._config.read(configFile)
self._setup = Setup(self._config)
self._loginInfo = LoginInfo('wave-login', self._config)
self._wave_connector = WaveConnector(self._loginInfo, self._setup.auth_url, self._setup.resource_url)
self._producerConfigs = []
for name in self._setup.producers:
producerConfig = ProducerConfig(
name.strip(), self._config, self._setup.rootPath, self._setup.dataFolder)
self._producerConfigs.append(producerConfig)
def _waveUploader(self, loginInfo, dataset_id, dataset, uploadMode, dataConfig, setup):
"""
this method is a stub methd for tests
"""
return WaveUploader(loginInfo, dataset_id, dataset, uploadMode, dataConfig, setup)
def execute(self):
"""
Execute the Health Service Upload by given an starting timestamp with int type
:return: True for running success, False for failure
"""
try:
self._wave_connector.login()
for producerConfig in self._producerConfigs:
producer = producerConfig.producer
print(type(producer))
subprocess.call(
'/bin/rm -rf ' + producerConfig.dataConfig.dataFolder + "/*.csv", shell=True)
Logger.logger.info('pull from ' + producer.name())
producer.pull(producerConfig.dataConfig.dataFolder)
Logger.logger.info('upload dataset ' + producerConfig.dataset)
waveUploader = self._waveUploader(
self._wave_connector,
producerConfig.data_id,
producerConfig.dataset,
self._uploadMode,
producerConfig.dataConfig,
self._setup)
waveUploader.uploadCsv(producerConfig.data_id)
return True
except Exception as e:
print(e)
info = "Error while executing."
exception_handler(info, e)
return False
def __init__(self, config_file):
parser = ConfigParser()
parser.optionxform = str
config_file = config_file
Logger.logger.info(config_file)
parser.read(config_file)
database_conf = DatabaseConfig(parser)
self._db_conf = database_conf
self.template_path = parser.get("Paths", 'template_path')
self.saql_path = parser.get("Paths", 'saql_path')
self.environment = parser.get('Wave_Config', 'environment')
download = parser.get('Paths', 'download')
output = parser.get('Paths', 'output')
self.download = download + "/" + self.environment
self.output = output + "/" + self.environment
# login and config
self.login_config = LoginInfo('login', parser)
self.wave_config = DashboardInfo(parser)
self._ui_mgr = UiManager(self._db_conf, self.output,
self.template_path, self.saql_path,
self.download, self.environment)
self._entity_service = EntityService(self._db_conf)
self.conn = WaveConnector(self.login_config, self.wave_config.authUrl,
self.wave_config.resource_url)
self.conn.login()
def test_fetch_data_by_url(self, mock_session, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# fetch succeed
mock_session.return_value.status_code = 200
mock_session.return_value.json.return_value = {"foo": "bar"}
result = self.wave_connector.fetch_data_by_url("fake url")
self.assertIsNotNone(result)
self.assertEqual(result, {"foo": "bar"})
# fetch failed
mock_session.return_value.status_code = 404
mock_session.return_value.json.return_value = {"foo": "bar"}
http_error = requests.exceptions.RequestException
mock_session.return_value.raise_for_status.side_effect = http_error
with self.assertRaises(SystemExit) as cm:
self.wave_connector.fetch_data_by_url("fake url")
self.assertEqual(1, mock_session.raise_for_status.call_count)
self.assertEqual(0, mock_session.json.call_count)
self.assertEqual(cm.exception.code, 1)
def test_delete_dashboard_by_id(self, mock_session, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# create succeed
mock_session.return_value.status_code = 200
result = self.wave_connector.delete_dashboard_by_id(
"fake-dashboard-id")
self.assertIsNotNone(result)
self.assertEqual(result, 200)
# create failed
mock_session.return_value.status_code = 404
http_error = requests.exceptions.RequestException
mock_session.return_value.raise_for_status.side_effect = http_error
with self.assertRaises(SystemExit) as cm:
self.wave_connector.delete_dashboard_by_id("fake-dashboard-id")
self.assertEqual(1, mock_session.raise_for_status.call_count)
self.assertEqual(0, mock_session.json.call_count)
self.assertEqual(cm.exception.code, 1)
def test_login(self, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
http_error = requests.exceptions.RequestException
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
mock_post.return_value.raise_for_status.side_effect = http_error
# login succeed
wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
wave_connector.login()
self.assertEqual(0, mock_post.raise_for_status.call_count)
# login failed
mock_post.return_value.status_code = 404
with self.assertRaises(SystemExit) as cm:
wave_connector = WaveConnector(self.mock_login_config,
" http://test.com", "/resource")
wave_connector.login()
self.assertEqual(1, mock_post.raise_for_status.call_count)
self.assertEqual(cm.exception.code, 1)
def __init__(self, configFile, mode):
"""
Constructor for wave data uploader.
:param configFile: config file name
"""
self._uploadMode = mode
self._config = ConfigParser()
self._config.read(configFile)
self._setup = Setup(self._config)
self._loginInfo = LoginInfo('wave-login', self._config)
self._wave_connector = WaveConnector(self._loginInfo, self._setup.auth_url, self._setup.resource_url)
self._producerConfigs = []
for name in self._setup.producers:
producerConfig = ProducerConfig(
name.strip(), self._config, self._setup.rootPath, self._setup.dataFolder)
self._producerConfigs.append(producerConfig)
def test_fetch_data_by_name(self, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# create succeed
# mock_session.return_value.status_code = 200
# mock_session.return_value.json.return_value = {"foo": "bar"}
self.wave_connector.fetch_data_by_url = mock.Mock(return_value={
"fake-category": ({
"label": "fake-name"
}, {
"label2": "fake-name2"
})
})
result = self.wave_connector.fetch_data_by_name(
"fake-category", "fake-name")
self.assertIsNotNone(result)
self.assertEqual(result, {"label": "fake-name"})
# create failed
http_error = requests.exceptions.RequestException
self.wave_connector.fetch_data_by_url = mock.Mock(
side_effect=http_error)
with self.assertRaises(SystemExit) as cm:
self.wave_connector.fetch_data_by_name("fake-category",
"fake-name")
self.assertEqual(cm.exception.code, 1)
class WaveUiProcessor:
def __init__(self, config_file):
parser = ConfigParser()
parser.optionxform = str
config_file = config_file
Logger.logger.info(config_file)
parser.read(config_file)
database_conf = DatabaseConfig(parser)
self._db_conf = database_conf
self.template_path = parser.get("Paths", 'template_path')
self.saql_path = parser.get("Paths", 'saql_path')
self.environment = parser.get('Wave_Config', 'environment')
download = parser.get('Paths', 'download')
output = parser.get('Paths', 'output')
self.download = download + "/" + self.environment
self.output = output + "/" + self.environment
# login and config
self.login_config = LoginInfo('login', parser)
self.wave_config = DashboardInfo(parser)
self._ui_mgr = UiManager(self._db_conf, self.output,
self.template_path, self.saql_path,
self.download, self.environment)
self._entity_service = EntityService(self._db_conf)
self.conn = WaveConnector(self.login_config, self.wave_config.authUrl,
self.wave_config.resource_url)
self.conn.login()
def delete_dashboard_list(self, dashboard_list):
"""
Method removes all dashboards in provided list and associated dashboard-page from database
:param dashboard_list:
:return:
"""
for d in dashboard_list:
logger.info("Deleting dashboard %s (%s)" %
(d.display_name, d.wave_id))
response = self.conn.delete_dashboard_by_id(d.wave_id)
logger.info(response)
logger.info("Removing dashboard %s (%s) from database" %
(d.display_name, d.wave_id))
containers = self._entity_service.get_dashboard_pages(d.wave_id)
for c in containers:
self._entity_service.delete_relationship_by_id(
d.wave_id, c.name)
self._entity_service.delete_dashboard_by_id(d.wave_id)
logger.info("Removed from database!")
def upload_single_dashboard(self, json_content, prev_wave_id):
"""
Method uploads dashboard to Wave and updates dashboard with valid Wave ID in database
:param json_content:
:param prev_wave_id:
:return:
"""
response = self.conn.create_dashboard(json.loads(json_content))
new_wave_id = response["id"]
url = response["url"]
folder_id = response["folder"]["id"]
logger.info("Dashboard %s (%s) created successfully" %
(response["name"], new_wave_id))
# get info from original wave_id
db_dashboard = self._entity_service.get_dashboard_by_id(prev_wave_id)
# add new dashboard to DB
self._entity_service.add_new_dashboard(
Dashboard(new_wave_id, db_dashboard.dashboard_name,
db_dashboard.dashboard_type, db_dashboard.display_name,
url, db_dashboard.link_var, db_dashboard.group_name,
folder_id, self.environment))
db_pages = self._entity_service.get_dashboard_pages(prev_wave_id)
# update all the relationships
for c in db_pages:
self._entity_service.add_relationship(new_wave_id, c.name)
self._entity_service.delete_relationship_by_id(
prev_wave_id, c.name)
self._entity_service.delete_dashboard_by_id(prev_wave_id)
logger.info("Updated database")
def create_dashboards(self):
"""
Method uploads new dashboard to Wave if it does not exist and updates existing dashboard otherwise
:return:
"""
dashboards = self._ui_mgr.get_dashboards(self.environment)
for ds in dashboards:
json_content = self._ui_mgr.generate_json(ds.dashboard_name,
ds.env)
try:
# adding to existing dashboard (valid wave_id)
# create_input_object = {'folder': ds.folder_id, 'name': ds.dashboard_name, 'description': ds.display_name, 'state': json.loads(json_content)}
# pretty_print(load_json_from(json_content))
response_ds = self.conn.create_dashboard(
data=load_json_from(json_content))
pretty_print(response_ds, 1)
logger.info(
"Dashboard %s (%s) updated ID %s" %
(ds.display_name, ds.dashboard_name, response_ds['id']))
ds.wave_id = response_ds['id']
ds.url = response_ds['url']
self._ui_mgr.save_dashboard(ds)
except requests.exceptions.RequestException:
# Wave ID from dashboard in DB does not exist
logger.info("Creating dashboard")
# self.upload_single_dashboard(json_content, ds.wave_id)
logger.info("<Response [201]>")
def update_dashboards(self):
"""
Method updates existing dashboards with generated JSON from metadata
:return:
"""
dashboards = self._ui_mgr.get_dashboards(self.environment)
for d in dashboards:
json_test = self._ui_mgr.generate_json(d.dashboard_name, d.env)
response = self.conn.update_dashboard_by_id(
d.wave_id, json.loads(json_test))
logger.info(response)
logger.info("Dashboard %s (%s) updated" %
(d.display_name, d.wave_id))
def delete_dashboards(self, dashboard_name=""):
"""
Method deletes dashboard by its name if provided or all dashboards otherwise
:param dashboard_name:
:return:
"""
dashboards = self._ui_mgr.get_dashboards(self.environment)
if dashboard_name:
dashboards = filter(
lambda x: x.get_dashboard_name() == dashboard_name, dashboards)
self.delete_dashboard_list(dashboards)
else:
self.delete_dashboard_list(dashboards)
def download_dashboards(self):
"""
Method saves JSON file of all dashboards that were uploaded/updated in the same session
:return:
"""
today = datetime.now()
dashboards = self._ui_mgr.get_dashboards(self.environment)
for d in dashboards:
file_name = self.output + "/" + d.display_name + "_dashboard.json"
content = read_file(file_name)
download_path = self.download + "/" + d.display_name + "_" + d.env + "_" + today.strftime(
'%Y%m%d%H%M') + "_dashboard.json"
save_file(download_path, content)
logger.info("Dashboard %s (%s) saved successfully" %
(d.display_name, d.wave_id))
class WaveConnectorTestSuite(unittest.TestCase):
"""Utils test cases."""
def setUp(self):
self.wave_connector = None
self.mock_login_config = mock.Mock(clientID="fake_clientID",
clientSecret="fake_clientSecret",
grantType="fake_grantType",
username="******",
password="******")
self.mock_dashboard_config = mock.Mock(
resource_url="fake_resource_url")
@mock.patch('wave_common.wave_connector.requests.post')
def test_login(self, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
http_error = requests.exceptions.RequestException
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
mock_post.return_value.raise_for_status.side_effect = http_error
# login succeed
wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
wave_connector.login()
self.assertEqual(0, mock_post.raise_for_status.call_count)
# login failed
mock_post.return_value.status_code = 404
with self.assertRaises(SystemExit) as cm:
wave_connector = WaveConnector(self.mock_login_config,
" http://test.com", "/resource")
wave_connector.login()
self.assertEqual(1, mock_post.raise_for_status.call_count)
self.assertEqual(cm.exception.code, 1)
@mock.patch('wave_common.wave_connector.requests.post')
@mock.patch('wave_common.wave_connector.requests.Session.get')
def test_fetch_data_by_url(self, mock_session, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# fetch succeed
mock_session.return_value.status_code = 200
mock_session.return_value.json.return_value = {"foo": "bar"}
result = self.wave_connector.fetch_data_by_url("fake url")
self.assertIsNotNone(result)
self.assertEqual(result, {"foo": "bar"})
# fetch failed
mock_session.return_value.status_code = 404
mock_session.return_value.json.return_value = {"foo": "bar"}
http_error = requests.exceptions.RequestException
mock_session.return_value.raise_for_status.side_effect = http_error
with self.assertRaises(SystemExit) as cm:
self.wave_connector.fetch_data_by_url("fake url")
self.assertEqual(1, mock_session.raise_for_status.call_count)
self.assertEqual(0, mock_session.json.call_count)
self.assertEqual(cm.exception.code, 1)
@mock.patch('wave_common.wave_connector.requests.post')
@mock.patch('wave_common.wave_connector.requests.Session.get')
def test_fetch_dashboard_by_id(self, mock_session, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# fetch succeed
mock_session.return_value.status_code = 200
mock_session.return_value.json.return_value = {"foo": "bar"}
result = self.wave_connector.fetch_dashboard_by_id("fake-dashboard-id")
self.assertIsNotNone(result)
self.assertEqual(result, {"foo": "bar"})
# fetch failed
mock_session.return_value.status_code = 404
mock_session.return_value.json.return_value = {"foo": "bar"}
http_error = requests.exceptions.RequestException
mock_session.return_value.raise_for_status.side_effect = http_error
with self.assertRaises(SystemExit) as cm:
self.wave_connector.fetch_dashboard_by_id("fake-dashboard-id")
self.assertEqual(1, mock_session.raise_for_status.call_count)
self.assertEqual(0, mock_session.json.call_count)
self.assertEqual(cm.exception.code, 1)
@mock.patch('wave_common.wave_connector.requests.post')
@mock.patch('wave_common.wave_connector.requests.Session.post')
def test_create_dashboard(self, mock_session, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# create succeed
mock_session.return_value.status_code = 200
mock_session.return_value.json.return_value = {"foo": "bar"}
result = self.wave_connector.create_dashboard("fake-dashboard-data")
self.assertIsNotNone(result)
self.assertEqual(result, {"foo": "bar"})
# create failed
mock_session.return_value.status_code = 404
mock_session.return_value.json.return_value = {"foo": "bar"}
http_error = requests.exceptions.RequestException
mock_session.return_value.raise_for_status.side_effect = http_error
with self.assertRaises(SystemExit) as cm:
self.wave_connector.create_dashboard("fake-dashboard-data")
self.assertEqual(1, mock_session.raise_for_status.call_count)
self.assertEqual(0, mock_session.json.call_count)
self.assertEqual(cm.exception.code, 1)
@mock.patch('wave_common.wave_connector.requests.post')
@mock.patch('wave_common.wave_connector.requests.Session.patch')
def test_update_dashboard_by_id(self, mock_session, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# create succeed
mock_session.return_value.status_code = 200
result = self.wave_connector.update_dashboard_by_id(
"fake-id", "fake-dashboard-data")
self.assertIsNotNone(result)
self.assertEqual(result, 200)
# create failed
mock_session.return_value.status_code = 404
http_error = requests.exceptions.RequestException
mock_session.return_value.raise_for_status.side_effect = http_error
with self.assertRaises(SystemExit) as cm:
self.wave_connector.update_dashboard_by_id("fake-id",
"fake-dashboard-data")
self.assertEqual(1, mock_session.raise_for_status.call_count)
self.assertEqual(0, mock_session.json.call_count)
self.assertEqual(cm.exception.code, 1)
@mock.patch('wave_common.wave_connector.requests.post')
@mock.patch('wave_common.wave_connector.requests.Session.delete')
def test_delete_dashboard_by_id(self, mock_session, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# create succeed
mock_session.return_value.status_code = 200
result = self.wave_connector.delete_dashboard_by_id(
"fake-dashboard-id")
self.assertIsNotNone(result)
self.assertEqual(result, 200)
# create failed
mock_session.return_value.status_code = 404
http_error = requests.exceptions.RequestException
mock_session.return_value.raise_for_status.side_effect = http_error
with self.assertRaises(SystemExit) as cm:
self.wave_connector.delete_dashboard_by_id("fake-dashboard-id")
self.assertEqual(1, mock_session.raise_for_status.call_count)
self.assertEqual(0, mock_session.json.call_count)
self.assertEqual(cm.exception.code, 1)
@mock.patch('wave_common.wave_connector.requests.post')
# @mock.patch('shd.wave_connector.requests.Session.get')
def test_fetch_data_by_name(self, mock_post):
response = {
"access_token": "fake_token",
"instance_url": "fake_url",
"token_type": "fake_type"
}
mock_post.return_value.status_code = 200
mock_post.return_value.json.return_value = response
self.wave_connector = WaveConnector(self.mock_login_config,
"http://test.com", "/resource")
# create succeed
# mock_session.return_value.status_code = 200
# mock_session.return_value.json.return_value = {"foo": "bar"}
self.wave_connector.fetch_data_by_url = mock.Mock(return_value={
"fake-category": ({
"label": "fake-name"
}, {
"label2": "fake-name2"
})
})
result = self.wave_connector.fetch_data_by_name(
"fake-category", "fake-name")
self.assertIsNotNone(result)
self.assertEqual(result, {"label": "fake-name"})
# create failed
http_error = requests.exceptions.RequestException
self.wave_connector.fetch_data_by_url = mock.Mock(
side_effect=http_error)
with self.assertRaises(SystemExit) as cm:
self.wave_connector.fetch_data_by_name("fake-category",
"fake-name")
self.assertEqual(cm.exception.code, 1)
if __name__ == '__main__':
unittest.main()