class ReportValidator(Report, object):
"""
Class for setting permission
"""
get_data = True
def __init__(self):
super(ReportValidator, self).__init__("normal", None, None)
def init(self, id):
# init element
super(ReportValidator, self).init(id)
self._date_unformatter = DateUnformat()
# do not save instances
self._data['report_save_historical_instances_ind'] = u'N'
# json file operation wrapper
self._jfile = JFileValidator(self._path, self._id, self._data)
self._meas_interval = None
if self.get_data:
#get validation settings from valid.json generated by editor
self.validation_data = self._jfile.get_validation_data()
if self.validation_data['measurement_interval_id']:
res = self._db.Query("""SELECT interval_unit
FROM measurement_interval
WHERE
measurement_interval_id =%s""", (self.validation_data['measurement_interval_id'], ))
if res:
self._meas_interval = self._db.record[0]['interval_unit']
self.validation_data['data_fetch_command'] = self.validation_data['data_fetch_command'].strip().replace('\r\n','\n')
if self.validation_data['source_database_connection_id']:
self._data['source_database_connection_id'] = int(self.validation_data['source_database_connection_id'])
else:
self._data['source_database_connection_id'] = 0
if 'plugin_connection_profile_id' in self.validation_data and self.validation_data['plugin_connection_profile_id']:
self._data['plugin_connection_profile_id'] = int(self.validation_data['plugin_connection_profile_id'])
else:
self._data['plugin_connection_profile_id'] = 0
self._data['data_fetch_command'] = self.validation_data['data_fetch_command']
self._data['data_fetch_method'] = self.validation_data['data_fetch_method']
if self.validation_data['web_service_credentials_id']:
self._data['web_service_credentials_id'] = int(self.validation_data['web_service_credentials_id'])
else:
self._data['web_service_credentials_id'] = 0
if self.validation_data['measurement_interval_id']:
self._data['measurement_interval_id'] = int(self.validation_data['measurement_interval_id'])
else:
self._data['measurement_interval_id'] = 0
# set report segment
if self.validation_data['segment_id'] and int(self.validation_data['segment_id']):
self._data['segment_id'] = int(self.validation_data['segment_id'])
else:
self._data['segment_id'] = 0
# get segment data
self._segment = self._get_segment()
self._segment_values = self._get_segment_values()
# get validation segment value id and set it as active
if self.validation_data['segment_value_id'] and int(self.validation_data['segment_value_id']):
segment_value_id = int(self.validation_data['segment_value_id'])
else:
segment_value_id = 0
# get check current segment value to be in report segments
if segment_value_id and self._segment and any(segment['segment_value_id'] == segment_value_id for segment in self._segment_values):
self._segment_value = list(segment for segment in self._segment_values if segment['segment_value_id'] == segment_value_id)[0]
elif self._segment_values and self._segment:
raise Exception("segment_value_id does not match segment_id")
else:
self._segment_value = []
if self._segment_value:
self._segment_value_id = self._segment_value['segment_value_id']
else:
self._segment_value_id = 0
self._jfile.set_segment_value(self._segment_value)
self._jfile.set_segment(self._segment)
def _check_pre_saved_settings(self, fetch_settings):
if (fetch_settings and
fetch_settings['sql'] and
fetch_settings['sql'] == self._data['data_fetch_command'] and
fetch_settings['source_database_connection_id'] == self._data['source_database_connection_id'] and
fetch_settings['segment_id'] == self._data['segment_id']):
return True
return False
def get_saved_instance(self):
"""
get clear data for building instance
"""
# data built by generator. it's actual data
generation_fetch_settings = self._jfile.get_generation_fetch_settings()
# data built by last validation
validation_fetch_settings = self._jfile.get_validation_fetch_settings()
# data built by last validation which was saved
last_validation_fetch_settings = self._jfile.get_last_validation_fetch_settings()
pre_saved_info = []
data = {'instance': None, 'meas_time': None}
meas_time = None
if self.validation_data['measurement_time']:
meas_time = self._date_unformatter.unformat(self.validation_data['measurement_time'])
if not meas_time:
raise Exception("Incorrect measurement_time date format.")
if self._check_pre_saved_settings(generation_fetch_settings):
# take generated
if meas_time:
res = self._db.Query("""SELECT report_data_set_instance_id
FROM report_data_set_instance
WHERE
`element_id`=%s
AND segment_value_id = %s
AND measurement_time = %s
ORDER BY measurement_time DESC
LIMIT 0, 1""",(self._id, self._segment_value_id, meas_time))
if res:
# has meas time
data_set_instance = self._db.record[0]
generation_fetch_settings['saved_data'] = self._jfile.get_generated_dataset_instance(data_set_instance['report_data_set_instance_id'])
else:
# not has
generation_fetch_settings['saved_data'] = None
else:
generation_fetch_settings['saved_data'] = self._jfile.get_generated_dataset()
generation_fetch_settings['source'] = 'generated'
pre_saved_info.append(generation_fetch_settings)
if self._check_pre_saved_settings(validation_fetch_settings):
#take validated
validation_fetch_settings['saved_data'] = self._jfile.get_validated_dataset()
validation_fetch_settings['source'] = 'validated'
pre_saved_info.append(validation_fetch_settings)
if self._check_pre_saved_settings(last_validation_fetch_settings):
# take last saved validated
last_validation_fetch_settings['saved_data'] = self._jfile.get_last_validated_dataset()
last_validation_fetch_settings['source'] = 'last_saved'
pre_saved_info.append(last_validation_fetch_settings)
# no validated query at all
if not pre_saved_info:
raise Exception("Please validate query first")
# get pre-saved items with data
pre_saved_info_with_data = [pre_saved for pre_saved in pre_saved_info if pre_saved['saved_data']]
# sort by generation time
pre_saved_info_with_data.sort(key=lambda item: item['saved_data']['generation_time'], reverse = True)
# check validation measurement time
#if self._data['data_fetch_method'] == 'sql':
if self.validation_data['measurement_time']:
# is measurement time is specified take only exact data
pre_saved_info_with_data = [pre_saved for pre_saved in pre_saved_info_with_data if self._date_unformatter.unformat(pre_saved['saved_data']['meas_time']) == meas_time]
else:
# is no measurement time take only actual data
# check if generated data exists
"""
pre_saved_info_generated = [pre_saved for pre_saved in pre_saved_info_with_data if pre_saved['source'] == 'generated']
if pre_saved_info_generated:
#use actual generated data'
# use generated data as it's actual
pre_saved_info_with_data = pre_saved_info_generated
else:
"""
if True:
#use filter to detect actual data
# filter actual data
expired_date = None
if self._data['max_time_before_expired_sec']:
#'use meas interval', self._data['max_time_before_expired_sec']
expired_date = datetime.datetime.now() - datetime.timedelta(seconds = self._data['max_time_before_expired_sec'])
elif self._meas_interval:
# 'use meas interval', self._meas_interval
if self._meas_interval == 'minute':
expired_date = datetime.datetime.now() - datetime.timedelta(minutes = 10)
elif self._meas_interval == 'hour':
expired_date = datetime.datetime.now() - datetime.timedelta(hours = 1)
elif self._meas_interval == 'day':
expired_date = datetime.datetime.now() - datetime.timedelta(days = 1)
elif self._meas_interval == 'week':
expired_date = datetime.datetime.now() - datetime.timedelta(weeks = 1)
elif self._meas_interval == 'month':
expired_date = datetime.datetime.now() - datetime.timedelta(days = 30)
elif self._meas_interval == 'quarter':
expired_date = datetime.datetime.now() - datetime.timedelta(days = 90)
elif self._meas_interval == 'year':
expired_date = datetime.datetime.now() - datetime.timedelta(days = 365)
else:
expired_date = None
# expired_date
if expired_date:
pre_saved_info_with_data = [pre_saved for pre_saved in pre_saved_info_with_data if self._date_unformatter.unformat(pre_saved['saved_data']['generation_time']) >= expired_date]
if pre_saved_info_with_data:
# if pre-saved data exists
self._get_outer_connection()
data['instance'] = self._outer_conn.parse_collected_data(simplejson.loads(pre_saved_info_with_data[0]['saved_data']['instance']))
data['meas_time'] = self._date_unformatter.unformat(pre_saved_info_with_data[0]['saved_data']['meas_time'])
else:
# no actual pre-saved data, let's fetch it
instance, meas_time = self._fetch_data_from_source()
data['instance'] = instance
data['meas_time'] = meas_time
# """
# #check generation sql to be equal current validation sql
# if not data['instance'] and \
# generation_fetch_settings and \
# generation_fetch_settings['sql'] and \
# generation_fetch_settings['sql'] == self.validation_data['data_fetch_command']:
# saved_data = self._jfile.get_generated_dataset()
# if saved_data:
# data = saved_data
#
# #check last validation sql to be equal current validation sql
# if not data['instance'] and \
# validation_fetch_settings and \
# validation_fetch_settings['sql'] and \
# validation_fetch_settings['sql'] == self.validation_data['data_fetch_command']:
# saved_data = self._jfile.get_validated_dataset()
# if saved_data:
# data = saved_data
#
# #check last saved validation sql to be equal current validation sql
# if not data['instance'] and \
# last_validation_fetch_settings and \
# last_validation_fetch_settings['sql'] and \
# last_validation_fetch_settings['sql'] == self.validation_data['data_fetch_command']:
# saved_data = self._jfile.get_last_validated_dataset()
# if saved_data:
# data = saved_data
#
# # no data found get by the same sql query as current validation sql.
# # raise exception - sql query must be validated
# if data['instance'] is None:
# raise Exception("Please validate query first")
# """
return data['instance'], data['meas_time']
def report_generation(self):
"""
generate all enabled report elements: dataset, pivots, charts
"""
# get instance
instance, meas_time = self.get_saved_instance()
self._jfile.set_meas_time(meas_time)
# create dataset instance
data_set_instance = self._process_instance(instance, meas_time, update_columns=False, write_clear_headers=True, segment_value = self._segment_value)
fetched_rows = len(instance['data'])
#prepare data for charts
all_data = {0: data_set_instance.get_formatted_header_rows()}
# create all pivots
for pivot in self._pivots:
data_set_pivot_instance = self._process_pivot(pivot, data_set_instance)
all_data[pivot['report_data_set_pivot_id']] = data_set_pivot_instance.get_formatted_header_rows()
# create all charts
for chart in self._charts:
if chart['report_data_set_pivot_id']:
#data_index = chart['report_data_set_pivot_id']
#data_chart = self._strip_total(chart, all_data[data_index])
data_chart = all_data[chart['report_data_set_pivot_id']]
else:
data_chart = all_data[0]
is_index = False
report_chart = ReportChart(chart['report_data_set_chart_id'],
self._id,
self._segment_value_id,
meas_time,
0,
data_chart,
self._jfile,
'large',
is_index,
self._data['preview_display_format_string'],
self._formatter)
report_chart.generateChart()
self._make_meta()
return fetched_rows
def data_generation(self):
"""
generate dataset
"""
# get instance
instance, meas_time = self.get_saved_instance()
# create dataset instance
self._process_instance(instance, meas_time, update_columns = False, write_clear_headers = True, segment_value = self._segment_value)
fetched_rows = len(instance['data'])
return fetched_rows
def _fetch_data_from_source(self):
#get last measurement time
if self.validation_data['last_measurement_time']:
# try to parse it from valid.json
last_meas_time = self._date_unformatter.unformat(self.validation_data['last_measurement_time'])
if not last_meas_time:
raise Exception("Incorrect last_measurement_time date format.")
else:
# get from db
last_meas_time = self._get_last_meas_time()
if self.validation_data['measurement_time']:
# try to parse it from valid.json
meas_time = self._date_unformatter.unformat(self.validation_data['measurement_time'])
if not meas_time:
raise Exception("Incorrect measurement_time date format.")
else:
meas_time = ''
#self._outer_conn = self._get_outer_connection()
self._get_outer_connection(get_meas_time_conn=False)
# get instance
if not meas_time:
meas_time = last_meas_time
# if self._data['data_fetch_method'] == 'web service':
# meas_times = self._get_meas_times_web_service(last_meas_time)
# if meas_times['data']:
# meas_time = meas_times['data'][-1][0]
#instance = self._get_instance(meas_time, self._segment_value, last_meas_time)
#instance = self._get_instance(meas_time, self._segment_value)
instance = self._get_instance(meas_time)
json_instance = self._outer_conn.get_json_result()
# save data fetch
self._jfile.save_data_fetch(
{'instance': json_instance,
'meas_time': meas_time.strftime('%Y-%m-%d %H:%M:%S'),
'generation_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')})
# save sql query
self._jfile.save_fetch_settings({'sql': self._data['data_fetch_command'],
'segment_id': self._data['segment_id'],
'source_database_connection_id': self._data['source_database_connection_id'],
})
return instance, meas_time
def data_fetch(self):
"""
fetch data from outer source
"""
instance, meas_time = self._fetch_data_from_source()
# process instance and generate dataset json
self._process_instance(instance, meas_time, update_columns=False, write_clear_headers=True, segment_value=self._segment_value)
fetched_rows = len(instance['data'])
# """
# # save clear fetched data
# self._jfile.save_data_fetch(
# {'instance': json_instance,
# 'meas_time': meas_time.strftime('%Y-%m-%d %H:%M:%S'),
# 'generation_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')})
#
# # save sql query
# self._jfile.save_fetch_settings({'sql': self._data['data_fetch_command'],
# 'segment_id': self._data['segment_id'],
# 'source_database_connection_id': self._data['source_database_connection_id'],
# })
#
#
# """
return fetched_rows
def metadata_update(self):
"""
update report metadata
"""
# get instance
instance, meas_time = self.get_saved_instance()
if not instance:
raise Exception("Cannot update metadata. Dataset is empty.")
# force dataset class update titles
self._data['last_report_column_metadata_update_time'] = None
self._update_fetch_command_update_time()
# process instance
self._process_instance(instance, meas_time, update_columns=True, write_clear_headers=True, segment_value=self._segment_value)
# if exists validated data and sql then store them as last validated
last_validation_fetch_settings = self._jfile.get_validation_fetch_settings()
if last_validation_fetch_settings:
last_validated_data = self._jfile.get_validated_dataset()
if last_validated_data:
self._jfile.save_last_validated_data_fetch(last_validated_data)
self._jfile.save_last_validation_fetch_settings(last_validation_fetch_settings)
return 0
def pivot_generation(self, pivot_id):
"""
generate pivot and charts based on this pivot
"""
# take all the pivots including disabled
self._pivots = self._get_pivots(enabled_only = False)
# check if there any pivots
if not self._pivots:
raise Exception("Report has no pivots")
if not pivot_id:
raise Exception("Pivot id is not specified")
#get enabled charts
self._charts = self._get_charts()
# find specified pivot
pivots = filter(lambda pivot: pivot['report_data_set_pivot_id'] == pivot_id, self._pivots)
if not pivots:
raise Exception("Incorrect pivot id")
pivot = pivots[0]
# get all chart based on current pivot
pivot_charts = filter(lambda chart: chart['report_data_set_pivot_id'] == pivot_id, self._charts)
# get instance
instance, meas_time = self.get_saved_instance()
self._jfile.set_meas_time(meas_time)
if not instance:
raise Exception("Cannot create pivot. Dataset is empty.")
# create dataset instance
data_set_instance = self._process_instance(instance, meas_time, update_columns=False, write_clear_headers=True, segment_value=self._segment_value)
# create pivot instance
data_set_pivot_instance = self._process_pivot(pivot, data_set_instance)
#prepare pivot data for charts
pivot_formatted_header_rows = data_set_pivot_instance.get_formatted_header_rows()
for pivot_chart in pivot_charts:
#data_chart = self._strip_total(pivot_chart, pivot_formatted_header_rows)
data_chart = pivot_formatted_header_rows
report_chart = ReportChart(pivot_chart['report_data_set_chart_id'],
self._id,
self._segment_value_id,
meas_time,
0,
data_chart,
self._jfile,
'large',
False,
self._data['preview_display_format_string'],
self._formatter)
report_chart.generateChart()
return len(pivot_formatted_header_rows['rows'])
def chart_generation(self, chart_id):
"""
generate chart
"""
return self._chart_process(chart_id, 'generate')
def saving_chart(self, chart_id):
"""
populate chart 'row values'
"""
return self._chart_process(chart_id, 'populate')
def _chart_process(self, chart_id, command):
"""
process chart
"""
# take all the pivots including disabled
self._pivots = self._get_pivots(enabled_only=False)
# take all the charts including disabled
self._charts = self._get_charts(enabled_only=False)
# check if there any charts
if not self._charts:
raise Exception("report has no chart")
if not chart_id:
raise Exception("chart id is not specified")
# find specified chart
charts = filter(lambda chart: chart['report_data_set_chart_id'] == chart_id, self._charts)
if not charts:
raise Exception("Incorrect chart id")
chart = charts[0]
all_data = {}
# get instance
instance, meas_time = self.get_saved_instance()
self._jfile.set_meas_time(meas_time)
if not instance:
raise Exception("Cannot create chart. Dataset is empty.")
# create dataset instance
data_set_instance = self._process_instance(instance, meas_time, update_columns=False, write_clear_headers=True, segment_value=self._segment_value)
if chart['report_data_set_pivot_id']:
# get pivot data
index = chart['report_data_set_pivot_id']
pivots = filter(lambda pivot: pivot['report_data_set_pivot_id'] == index, self._pivots)
pivot = pivots[0]
data_set_pivot_instance = self._process_pivot(pivot, data_set_instance)
formatted_header_rows = data_set_pivot_instance.get_formatted_header_rows()
else:
# get dataset
index = 0
formatted_header_rows = data_set_instance.get_formatted_header_rows()
if not formatted_header_rows:
raise Exception("Cannot create chart. Dataset is empty.")
#prepare data for charts
all_data[index] = formatted_header_rows
if command == 'generate':
if chart['report_data_set_pivot_id']:
#data_index = chart['report_data_set_pivot_id']
#data_chart = self._strip_total(chart, all_data[data_index])
data_chart = all_data[chart['report_data_set_pivot_id']]
else:
data_chart = all_data[0]
is_index = False
report_chart = ReportChart(chart['report_data_set_chart_id'],
self._id,
self._segment_value_id,
meas_time,
0,
data_chart,
self._jfile,
'large',
is_index,
self._data['preview_display_format_string'],
self._formatter)
report_chart.generateChart()
elif command == 'populate':
self._populate_row_values([chart], all_data)
return 0
def save_validation_data(self):
self._jfile.save_validation_data()
# take all the pivots including disabled
self._pivots = self._get_pivots(enabled_only=False)
# take all the charts including disabled
self._charts = self._get_charts(enabled_only=False)
if self._charts:
all_data = {}
# get instance
instance, meas_time = self.get_saved_instance()
self._jfile.set_meas_time(meas_time)
if not instance:
raise Exception("Cannot create chart. Dataset is empty.")
# create dataset instance
data_set_instance = self._process_instance(instance, meas_time, update_columns=False, write_clear_headers=True, segment_value=self._segment_value)
# create all charts
for chart in self._charts:
if chart['report_data_set_pivot_id'] not in all_data:
# get pivot data
if chart['report_data_set_pivot_id']:
index = chart['report_data_set_pivot_id']
pivots = filter(lambda pivot: pivot['report_data_set_pivot_id'] == index, self._pivots)
pivot = pivots[0]
data_set_pivot_instance = self._process_pivot(pivot, data_set_instance)
formatted_header_rows = data_set_pivot_instance.get_formatted_header_rows()
else:
# get dataset
index = 0
formatted_header_rows = data_set_instance.get_formatted_header_rows()
if not formatted_header_rows:
raise Exception("Cannot create chart. Dataset is empty.")
#prepare data for charts
all_data[index] = formatted_header_rows
# create all charts
for chart in self._charts:
self._populate_row_values([chart], all_data)
return 0
def restore_validation_data(self):
self._jfile.restore_validation_data()
return 0
def _update_fetch_command_update_time(self):
"""
Update last_display_generation_time
"""
self._db.Query("""UPDATE dashboard_element
SET last_data_fetch_command_update_time = NOW()
WHERE
`element_id` = %s""", (self._id, ))
