def shouldNullify(exclusion: json, src_key: str) -> bool:
"""
shouldNullify
Check if the exclusion requires to nullify the measure
Parameter:
exclusion: json coming from exclusion table
src_key: key to check in the exclusion field
"""
try:
if exclusion is not None:
if (exclusion.__contains__(src_key) is True and exclusion[src_key]
== "null") or exclusion.__contains__(src_key) is False:
return True
return False
except Exception as e:
if e.__dict__.__len__() == 0 or "done" not in e.__dict__:
exception_type, exception_object, exception_traceback = sys.exc_info(
)
exception_info = e.__repr__()
filename = exception_traceback.tb_frame.f_code.co_filename
funcname = exception_traceback.tb_frame.f_code.co_name
line_number = exception_traceback.tb_lineno
e.info = {
"i": str(exception_info),
"n": funcname,
"f": filename,
"l": line_number,
}
e.done = True
raise e
def delKey(j: json, key: str):
"""
delKey
delete a key in json if exists
Parameter:
j: json data
key: key to delete if exists
"""
try:
if j.__contains__(key):
del j[key]
except Exception as e:
if e.__dict__.__len__() == 0 or "done" not in e.__dict__:
exception_type, exception_object, exception_traceback = sys.exc_info(
)
exception_info = e.__repr__()
filename = exception_traceback.tb_frame.f_code.co_filename
funcname = exception_traceback.tb_frame.f_code.co_name
line_number = exception_traceback.tb_lineno
e.info = {
"i": str(exception_info),
"n": funcname,
"f": filename,
"l": line_number,
}
e.done = True
raise e
def addJson(j: json, key: str, valeur: int):
"""
addJson
add a value to j[key]
Parameter:
j: json field
key: key to add
valeur: value to add
"""
try:
if j.__contains__(key) is False:
j[key] = 0
j[key] += valeur
except Exception as e:
if e.__dict__.__len__() == 0 or "done" not in e.__dict__:
exception_type, exception_object, exception_traceback = sys.exc_info(
)
exception_info = e.__repr__()
filename = exception_traceback.tb_frame.f_code.co_filename
funcname = exception_traceback.tb_frame.f_code.co_name
line_number = exception_traceback.tb_lineno
e.info = {
"i": str(exception_info),
"n": funcname,
"f": filename,
"l": line_number,
}
e.done = True
raise e
def loadDataInObs(
self,
my_measure: json,
obs_meteor: ObsMeteor,
target_key: str,
delta_values: json,
trace_flag: bool,
):
if my_measure.__contains__('xyz') is False:
# just to satisfy our parser... Will always fail
raise Exception('loadDataInDV', 'should be in virtual func')
def loadValuesFromCurrent(
self,
my_measure: json,
json_file_data: json,
measure_idx: int,
src_key: str,
target_key: str,
exclusion: json,
my_values: json,
trace_flag: bool,
):
if my_measure.__contains__('xyz') is False:
# just to satisfy our parser... Will always fail
raise Exception('loadDataInDV', 'should be in virtual func')
def loadMaxMinInObs(
self,
my_measure: json,
obs_meteor: ObsMeteor,
target_key: str,
delta_values: json,
my_values: json,
trace_flag: bool = False,
):
"""
loadMaxMinInObsInObservation
load in obs max/min value if present
update delta_values
calculus v2
"""
obs_j = obs_meteor.data.j
b_is_max = True
for maxmin_key in ['max', 'min']:
# is max or min needed for this measure
maxmin_suffix = '_' + maxmin_key
# if target_key == "rain_rate":
# target_key = "rain_rate"
if my_measure.__contains__(maxmin_key) is True and my_measure[maxmin_key] is True:
# propagate the check request
if my_values.get(target_key + '_check' + maxmin_suffix) is not None:
obs_j[target_key + '_check' + maxmin_suffix] = my_values[target_key + '_check' + maxmin_suffix]
if my_values.get(target_key + maxmin_suffix) is not None:
my_maxmin_value = my_values[target_key + maxmin_suffix]
my_maxmin_date = my_values[target_key + maxmin_suffix + '_time']
obs_j[target_key + maxmin_suffix] = my_maxmin_value
obs_j[target_key + maxmin_suffix + '_time'] = my_maxmin_date
delta_values[target_key + maxmin_suffix] = my_maxmin_value
delta_values[target_key + maxmin_suffix + '_time'] = my_maxmin_date
if my_values.get(target_key + maxmin_suffix + '_dir') is not None:
my_wind_dir = my_values[target_key + maxmin_suffix + '_dir']
obs_j[target_key + maxmin_suffix + '_dir'] = my_wind_dir
delta_values[target_key + maxmin_suffix + '_dir'] = my_wind_dir
b_is_max = not(b_is_max)
def loadDataInObs(
self,
my_measure: json,
obs_meteor: ObsMeteor,
target_key: str,
delta_values: json,
my_values: json,
trace_flag: bool,
):
"""
loadObservationDatarow
load Observation dataRow from json measures
load delta_values from json mesures
"""
obs_j = obs_meteor.data.j
if my_measure['target_key'] == 'barometer':
my_value_avg = my_values.get(target_key + '_a')
my_value_avg = my_values.get(target_key + '_a')
my_value_instant = my_values.get(target_key + '_i')
my_value_dir = my_values.get(target_key + '_di')
tmp_duration = my_values.get(target_key + '_du')
if isFlagged(my_measure['special'],
MeasureProcessingBitMask.MeasureIsOmm):
tmp_duration = 60
if my_value_avg is None:
if my_value_instant is None:
return
else:
my_value_avg = my_value_instant
# get our value to aggregate
if my_measure.__contains__('measureType'):
if my_measure['measureType'] == 'inst':
# for omm measure...
my_value_aggreg = my_value_instant
else:
my_value_aggreg = my_value_avg
else:
if my_value_instant is not None:
my_value_aggreg = my_value_instant
tmp_s = my_value_aggreg * tmp_duration
if my_value_aggreg is None:
# no data suitable for us
return
# get current values from our aggregations
tmp_s_old = tmp_duration_old = 0
if obs_j.get(target_key + '_s') is not None:
tmp_s_old = obs_j[target_key + '_s']
if tmp_s == tmp_s_old:
# no change on avg computation
return
tmp_duration_old = obs_j.get(target_key + '_duration')
if tmp_duration_old is None:
tmp_duration_old = tmp_duration
if tmp_duration != tmp_duration_old:
raise ('loadDataInObs', 'duration mitchmath for ' +
target_key + ': in obs' + str(tmp_duration_old) +
', in json: ' + str(tmp_duration))
delta_values[target_key + '_s_old'] = tmp_s_old
delta_values[target_key + '_duration_old'] = tmp_duration_old
# save data in dv
delta_values[target_key + '_s'] = tmp_s
delta_values[target_key + '_duration'] = tmp_duration
# save data in obs
obs_j[target_key] = my_value_aggreg
if my_value_dir is not None:
obs_j[target_key + '_dir'] = my_value_dir
def loadDVDataInAggregation(self,
my_measure: json,
m_stop_dat: datetime,
agg_deca: AggMeteor,
m_agg_j: json,
delta_values: json,
dv_next: json,
trace_flag: bool = False,
agg_suffix: str = '_sum'):
"""
loadDVDataInAggregation
Load one aggretation level with values from delta_values, update dv_next
parameters:
my_measure: measure definition
stop_dat: mesure stop_dat
agg_deca
m_agg_j: aggregations clause in json file
delta_values: json for forced values
dv_next: delta_values for next level
flag: True=insert, False=delete
"""
target_key = my_measure['target_key']
agg_j = agg_deca.data.j
agg_level = agg_deca.getLevelCode()
has_data = False
if target_key == 'rain_omm':
has_data = False
# load old measure values in case of an update of Observation, and only in agg_hour
tmp_sum_old = tmp_duration_old = 0
if delta_values.__contains__(target_key + '_sum_old'):
tmp_sum_old = delta_values[target_key + '_sum_old']
tmp_duration_old = delta_values[target_key + '_duration_old']
has_data = True
# ------------------------------------------------------------------
# get our new data
# 1 from dv[target_key + '_sum']
# 2 from m_agg_j[target_key + '_sum']
# last win
# ------------------------------------------------------------------
if delta_values.get('duration') is None:
tmp_duration = getAggDuration(agg_level[0],
agg_deca.data.start_dat)
else:
tmp_duration = float(delta_values["duration"])
# get our M_s from our delta_values
tmp_tmp = self.get_json_value(delta_values, target_key, [agg_suffix],
None)
if tmp_tmp is not None:
has_data = True
tmp_sum = float(tmp_tmp)
if delta_values.get(target_key + '_duration') is not None:
if isFlagged(my_measure['special'], MeasureProcessingBitMask.
MeasureIsOmm) and agg_level[0] == 'H':
tmp_duration = getAggDuration(agg_level[0],
agg_deca.data.start_dat)
else:
tmp_duration = delta_values[target_key + '_duration']
synos = [target_key]
if my_measure.get('syno') is not None:
for a_syno in my_measure['syno']:
synos.append(a_syno)
b_value_found = False
for a_key in synos:
if b_value_found:
continue
tmp_tmp = self.get_json_value(m_agg_j, a_key, [agg_suffix], True)
if tmp_tmp is not None:
has_data = True
tmp_sum = float(tmp_tmp)
if m_agg_j.__contains__(a_key + '_duration') is True:
tmp_duration = float(m_agg_j[a_key + '_duration'])
else:
tmp_duration = getAggDuration(agg_level[0],
agg_deca.data.start_dat)
if isFlagged(my_measure['special'],
MeasureProcessingBitMask.MeasureIsOmm):
tmp_duration = getAggDuration(agg_level[0],
agg_deca.data.start_dat)
b_value_found = True
# return if the aggregation should not be sent to upper levels
if has_data is False:
return
addJson(agg_j, target_key + agg_suffix, tmp_sum - tmp_sum_old)
addJson(agg_j, target_key + '_duration',
tmp_duration - tmp_duration_old)
tmp_sum_new = agg_j[target_key + agg_suffix]
tmp_duration_new = agg_j[target_key + '_duration']
if tmp_duration_new == 0:
# no duration, delete all keys
delKey(agg_j, target_key + agg_suffix)
delKey(agg_j, target_key + '_duration')
delta_values[target_key + '_delete_me'] = True
else:
# Add omm values in agg_hour
if isFlagged(my_measure['special'],
MeasureProcessingBitMask.MeasureIsOmm):
agg_j[target_key] = tmp_sum_new
if isFlagged(my_measure['special'],
MeasureProcessingBitMask.OnlyAggregateInHour) is True:
return
# propagate to next level if no limitation on aggregation level
dv_next[target_key + agg_suffix] = tmp_sum - tmp_sum_old
dv_next[target_key + '_duration'] = tmp_duration - tmp_duration_old
if delta_values.get('duration') is None:
dv_next['duration'] = tmp_duration
else:
dv_next["duration"] = delta_values['duration']
def _loadValuesFromCurrent(
self,
my_measure: json,
json_file_data: json,
measure_idx: int,
src_key: str,
target_key: str,
exclusion: json,
my_values: json,
key_suffix: str,
stop_dat: datetime,
trace_flag: bool,
):
"""
Load ou values in delta_value for value passing to loadDataInObs, and loadMaxMinInObsInObservation
"""
# b_exclu = True -> load data from exclusion, False -> normal processing
if src_key == 'rain_rate':
b_exclu = loadFromExclu(exclusion, src_key)
b_exclu = loadFromExclu(exclusion, src_key)
my_value_instant = my_value_avg = None
my_value_dir = None
if my_measure['target_key'] == 'barometer':
measure_type = 3
measure_type = 3
if my_measure.__contains__('measureType'):
if my_measure['measureType'] == 'avg':
measure_type = 1
elif my_measure['measureType'] == 'inst':
measure_type = 2
elif my_measure['measureType'] != 'both':
raise Exception('processJsonDataAvg::loadDataInObs', 'invalid measureType: ' + my_measure['measureType'] + ' for ' + src_key)
if b_exclu is False:
# load our data from the measure (json)
data_src = json_file_data['data'][measure_idx].get('current')
else:
data_src = exclusion
all_src_keys = [my_measure['src_key']]
if my_measure.get('syno') is not None:
for a_syno in my_measure['syno']:
all_src_keys.append(a_syno)
value_found = False
for a_srckey in all_src_keys:
if value_found:
continue
if data_src is not None:
if data_src.__contains__(a_srckey) and (measure_type & 2) == 2:
my_value_instant = self.get_json_value(data_src, a_srckey, [], True)
value_found = True
if data_src.__contains__(a_srckey + key_suffix) and (measure_type & 1) == 1:
my_value_avg = self.get_json_value(data_src, a_srckey + key_suffix, [], True)
value_found = True
if (isFlagged(my_measure['special'], MeasureProcessingBitMask.MeasureIsWind)):
if data_src.__contains__(a_srckey + '_dir'):
my_value_dir = data_src[a_srckey + '_dir']
elif data_src.__contains__(a_srckey + key_suffix + '_dir'):
my_value_dir = data_src[a_srckey + '_dir']
# init value instantaneout and avg
# measure_type I 1 I 2 I 3
# measure_avg (ma) I ma, then mi I None I ma, then mi
# measure_instant (mi) I None I mi, then ma I mi, then ma
if measure_type == 1:
if my_value_avg is None:
my_value_avg = my_value_instant
if measure_type == 1:
my_value_instant = None
if measure_type == 2:
if my_value_instant is None:
my_value_instant = my_value_avg
if measure_type == 2:
my_value_avg = None
if measure_type == 3:
if my_value_avg is None:
my_value_avg = my_value_instant
if my_value_instant is None:
my_value_instant = my_value_avg
if my_value_avg is not None:
my_values[target_key + '_a'] = my_value_avg
if my_value_instant is not None:
my_values[target_key + '_i'] = my_value_instant
if my_value_dir is not None:
my_values[target_key + '_di'] = my_value_dir
tmp_duration = int(json_file_data['data'][measure_idx]['current']['duration'])
if isFlagged(my_measure['special'], MeasureProcessingBitMask.MeasureIsOmm):
tmp_duration = 60
if tmp_duration != 0:
my_values[target_key + '_du'] = tmp_duration
# load max/min from json
for maxmin_key in ['max', 'min']:
if my_measure.__contains__(maxmin_key) is True and my_measure[maxmin_key] is True:
maxmin_suffix = '_' + maxmin_key
if data_src is not None and data_src.get(src_key + maxmin_suffix) is not None:
my_values[target_key + maxmin_suffix] = data_src[src_key + maxmin_suffix]
my_values[target_key + maxmin_suffix + '_time'] = data_src[src_key + maxmin_suffix + '_time']
if maxmin_key == 'max' and (isFlagged(my_measure['special'], MeasureProcessingBitMask.MeasureIsWind)):
if data_src is not None and data_src.get(src_key + maxmin_suffix + '_dir') is not None:
my_values[target_key + maxmin_suffix + '_dir'] = data_src[src_key + maxmin_suffix + '_dir']
else:
if my_value_avg is not None or my_value_instant is not None:
my_values[target_key + maxmin_suffix] = my_value_instant if my_value_instant is not None else my_value_avg
if data_src is not None and data_src.get(src_key + maxmin_suffix + "_time") is not None:
my_values[target_key + maxmin_suffix + '_time'] = data_src[src_key + maxmin_suffix + '_time']
else:
my_values[target_key + maxmin_suffix + '_time'] = stop_dat
def loadDVMaxMinInAggregation(
self,
my_measure: json,
m_stop_date: datetime,
agg_decas: AggMeteor,
m_agg_j: json,
delta_values: json,
dv_next: json,
trace_flag: bool = False,
):
"""
loadDVMaxMinInAggregation
load in all aggregations max/min
update dv_next for nest level
"""
target_key = my_measure['target_key']
idx_maxmin = 0
for maxmin_suffix in ['_max', '_min']:
idx_maxmin += 1
agg_j = agg_decas[idx_maxmin].data.j
maxmin_key = maxmin_suffix.split('_')[1]
if my_measure.__contains__(
maxmin_key) and my_measure[maxmin_key] is True:
new_calulated_maxmin = None
new_calulated_maxmin_dir = None
if target_key == "wind" and maxmin_key == 'max':
target_key = "wind"
# check if measure was deleted
if delta_values.__contains__(target_key +
'_delete_me') is True:
# measure was deleted previously
if agg_j.__contains__(target_key + maxmin_suffix):
# need to invalidate this value for next level
invalid_value = agg_j[target_key + maxmin_suffix]
dv_next[target_key + maxmin_suffix +
'_invalidate'] = invalid_value
dv_next[target_key + '_check' +
maxmin_suffix] = agg_j[target_key +
maxmin_suffix]
delKey(agg_j, target_key + maxmin_suffix)
delKey(agg_j, target_key + maxmin_suffix + '_time')
continue
# get the current max/min (the last load win)
# - load current value form delta_values
# - load current max/min from delta_values if given
# - load from "aggregations" clause in the json data file if given
if delta_values.__contains__(target_key):
new_calulated_maxmin = my_measure['dataType'](
delta_values[target_key])
new_calulated_maxmin_time = m_stop_date
if (isFlagged(my_measure['special'],
MeasureProcessingBitMask.MeasureIsWind)):
if delta_values.__contains__(target_key +
maxmin_suffix +
'_dir') is True:
new_calulated_maxmin_dir = float(
delta_values[target_key + maxmin_suffix +
'_dir'])
if delta_values.__contains__(target_key +
maxmin_suffix) is True:
new_calulated_maxmin = my_measure['dataType'](
delta_values[target_key + maxmin_suffix])
new_calulated_maxmin_time = delta_values[target_key +
maxmin_suffix +
'_time']
if (isFlagged(my_measure['special'],
MeasureProcessingBitMask.MeasureIsWind)):
if delta_values.__contains__(target_key +
maxmin_suffix +
'_dir') is True:
new_calulated_maxmin_dir = float(
delta_values[target_key + maxmin_suffix +
'_dir'])
if m_agg_j.__contains__(target_key + maxmin_suffix):
new_calulated_maxmin = my_measure['dataType'](
m_agg_j[target_key + maxmin_suffix])
new_calulated_maxmin_time = m_agg_j[target_key +
maxmin_suffix +
'_time']
if (isFlagged(my_measure['special'],
MeasureProcessingBitMask.MeasureIsWind)):
if m_agg_j.__contains__(target_key + maxmin_suffix +
'_dir') is True:
new_calulated_maxmin_dir = float(
m_agg_j[target_key + maxmin_suffix + '_dir'])
if new_calulated_maxmin is None:
# should never occurs...
continue
# load current max/min value from our aggregation
agg_maxmin = None
if agg_j.__contains__(target_key + maxmin_suffix):
agg_maxmin = agg_j[target_key + maxmin_suffix]
"""
invalidation decision tree for [field]_max
new_calulated_maxmin + agg_maxmin + former_maxmin_value + action
basic
10 + + No + update
10 + 5 + No + update
10 + 10 + No + pass
10 + 15 + No + pass
check_maxmin flag
10 + 5 + 5 + update -> auto
10 + 10 + 10 + pass -> auto
10 + 15 + 15 + recompute
"""
if delta_values.__contains__(target_key + '_check' +
maxmin_suffix):
if agg_maxmin is None:
raise Exception(
'loadDVMaxMinInAggregation',
'Invalidate and no data in aggregation...')
former_maxmin_value = delta_values[target_key + '_check' +
maxmin_suffix]
if maxmin_suffix == '_max':
if agg_maxmin > former_maxmin_value:
agg_maxmin = None
dv_next[target_key + '_check' +
maxmin_suffix] = former_maxmin_value
self.add_new_maxmin_fix(
target_key, maxmin_key,
my_measure['deca' + maxmin_suffix],
agg_decas[idx_maxmin], delta_values)
else:
if agg_maxmin < former_maxmin_value:
agg_maxmin = None
dv_next[target_key + '_check' +
maxmin_suffix] = former_maxmin_value
self.add_new_maxmin_fix(
target_key, maxmin_key,
my_measure['deca' + maxmin_suffix],
agg_decas[idx_maxmin], delta_values)
if agg_maxmin is None:
# force the update i agg_deca for our new_calulated_maxmin
if maxmin_suffix == '_min':
agg_maxmin = new_calulated_maxmin + 1
else:
agg_maxmin = new_calulated_maxmin - 1
# if (target_key == "wind"):
# if agg_j.__contains__(target_key + maxmin_suffix + '_time'):
# print(" wind_max: old_time=> " + str(agg_j[target_key + maxmin_suffix + '_time']) + ", new=> " + str(new_calulated_maxmin_time))
# print(" agg: id: " + str(agg_decas[idx_maxmin].data.id) + ', level: ' + agg_decas[idx_maxmin].agg_niveau + ', start_dat: ' + str(agg_decas[idx_maxmin].data.start_dat))
# else:
# print(" wind_max: old_time=> ** no data **, new=> " + str(new_calulated_maxmin_time))
# print(" agg: id: " + str(agg_decas[idx_maxmin].data.id) + ', level: ' + agg_decas[idx_maxmin].agg_niveau + ', start_dat: ' + str(agg_decas[idx_maxmin].data.start_dat))
# compare the measure data and current maxmin
b_change_maxmin = False
if maxmin_suffix == '_max' and agg_maxmin < new_calulated_maxmin:
# if (target_key == "wind"):
# print(' *** agg < new_calc')
b_change_maxmin = True
# code to keep the latest _max/min_time in case of equality
# if maxmin_suffix == '_max' and agg_maxmin == new_calulated_maxmin and str(agg_j[target_key + maxmin_suffix + '_time']) < str(new_calulated_maxmin_time):
# # if (target_key == "wind"):
# # print(' *** max agg_time < new_calc_time -> yes')
# b_change_maxmin = True
# if maxmin_suffix == '_max' and agg_maxmin == new_calulated_maxmin and str(agg_j[target_key + maxmin_suffix + '_time']) > str(new_calulated_maxmin_time):
# if (target_key == "wind"):
# print(' *** max agg_time > new_calc_time -> NO')
if maxmin_suffix == '_min' and agg_maxmin > new_calulated_maxmin:
b_change_maxmin = True
if maxmin_suffix == '_min' and agg_maxmin == new_calulated_maxmin and str(
agg_j[target_key + maxmin_suffix +
'_time']) > str(new_calulated_maxmin_time):
b_change_maxmin = True
if b_change_maxmin:
agg_j[target_key + maxmin_suffix] = new_calulated_maxmin
dv_next[target_key + maxmin_suffix] = new_calulated_maxmin
agg_j[target_key + maxmin_suffix +
'_time'] = new_calulated_maxmin_time
dv_next[target_key + maxmin_suffix +
'_time'] = new_calulated_maxmin_time
if (isFlagged(my_measure['special'],
MeasureProcessingBitMask.MeasureIsWind)):
if new_calulated_maxmin_dir is not None:
agg_j[target_key + maxmin_suffix +
'_dir'] = new_calulated_maxmin_dir
dv_next[target_key + maxmin_suffix +
'_dir'] = new_calulated_maxmin_dir
def _checkJsonOneItem(j: json, pid: int, meteor: str,
stop_dat_list: list) -> str:
try:
idx = 0
val_to_add = []
val_to_add_agg = []
val_to_add_val = []
j["poste_id"] = pid
if j.__contains__("meteor") is False or j["meteor"].__len__() == 0:
return "missing or invalid code meteor"
if j["meteor"] != meteor:
return "different code meteor: " + meteor + "/" + j["meteor"]
while idx < j["data"].__len__():
new_val = {}
new_val_agg = {}
new_val_val = {}
a_current = j["data"][idx].get("current")
tmp_stop_dat = j["data"][idx].get("stop_dat")
if a_current is None:
if (j["data"][idx].__contains__("aggregations") is False and
j["data"][idx].__contains__("validation") is False):
return (
"no current/aggregations/validation key in j.data[" +
str(idx) + "]")
else:
# check current key
if a_current.__contains__("duration") is False:
return "no duration in j.data[" + str(idx) + "].current"
measure_duration = a_current["duration"]
tmp_stop_dat = None
if j["data"][idx].__contains__("stop_dat") is False:
if j["data"][idx].__contains__("start_dat") is False:
return "no start and stop_dat in j.data[" + str(
idx) + "].current"
measure_duration = datetime.timedelta(
minutes=int(a_current["duration"]))
new_val = {
"k": "stop_dat",
"v": j["data"][idx]["start_dat"] + measure_duration,
"idx": idx,
}
val_to_add.append(new_val)
tmp_stop_dat = new_val["stop_dat"]
else:
tmp_stop_dat = j["data"][idx]["stop_dat"]
if str(tmp_stop_dat) in stop_dat_list:
return "stop_dat: " + str(tmp_stop_dat) + " present twice"
stop_dat_list.append(str(tmp_stop_dat))
if j["data"][idx].__contains__("start_dat") is False:
measure_duration = datetime.timedelta(
minutes=int(a_current["duration"]))
new_val = {
"k": "start_dat",
"v": j["data"][idx]["stop_dat"],
"idx": idx,
}
val_to_add.append(new_val)
for key in a_current.__iter__():
if str(key).endswith("_max") or str(key).endswith("_min"):
if a_current.__contains__(
key +
"time") is True and a_current.__contains__(
key + "_time") is False:
a_current[key + "_time"] = a_current[key + "time"]
new_val = {
"k": key + "_time",
"v": a_current[key + "time"],
"idx": idx,
"k2": "current",
}
val_to_add.append(new_val)
if a_current.__contains__(key + "_time") is False:
new_val = {
"k": key + "_time",
"v": j["data"][idx]["stop_dat"],
"idx": idx,
"k2": "current",
}
val_to_add.append(new_val)
if str(key).endswith("_sum"):
new_val = {
"k": str(key).replace("_sum", "_s"),
"v": a_current[key],
"idx": idx,
"k2": "current",
}
val_to_add.append(new_val)
if (str(key).endswith("_s")
and a_current.__contains__(key[:-4] + "_duration")
is False):
new_val = {
"k": key[:-4] + "_duration",
"v": measure_duration,
"idx": idx,
"k2": "current",
}
val_to_add.append(new_val)
# old specification...
if j["data"][idx]["current"].__contains__("aggregations"):
return "aggregations is under the key current, should be at same level"
if tmp_stop_dat is None:
return "no stop_dat, and no way to compute it"
all_aggreg = j["data"][idx].get("aggregations")
if all_aggreg is not None:
idx2 = 0
while idx2 < all_aggreg.__len__():
a_aggreg = j["data"][idx]["aggregations"][idx2]
if a_aggreg.__contains__("level") is False:
return ("no level in data[" + str(idx) +
"].aggregations[" + str(idx2) + "]")
lvl = a_aggreg["level"]
if (lvl != "H" and lvl != "D" and lvl != "M" and lvl != "Y"
and lvl != "A"):
return (lvl + " is invalid level in data[" + str(idx) +
"].aggregations[" + str(idx2) + "]")
for key in a_aggreg.__iter__():
if str(key).endswith(
"_sum") and str(key).endswith("_s") is False:
new_val_agg = {
"k": str(key).replace("_sum", "_s"),
"v": a_aggreg[key],
"idx": idx,
"idx2": idx2,
}
val_to_add_agg.append(new_val_agg)
if str(key).endswith("_max") or str(key).endswith(
"_min"):
if all_aggreg.__contains__(key + "_time") is False:
new_val_agg = {
"k": key + "_time",
"v": j["data"][idx]["stop_dat"],
"idx": idx,
"idx2": idx2,
}
val_to_add_agg.append(new_val_agg)
idx2 += 1
all_validations = j["data"][idx].get("validation")
if all_validations is not None:
idx2 = 0
while idx2 < all_validations.__len__():
a_aggreg = j["data"][idx]["validation"][idx2]
if a_aggreg.__contains__("level") is False:
return ("no level in data[" + str(idx) +
"].validation[" + str(idx2) + "]")
lvl = a_aggreg["level"]
if (lvl != "H" and lvl != "D" and lvl != "M" and lvl != "Y"
and lvl != "A"):
return (lvl + " is invalid level in data[" + str(idx) +
"].validation[" + str(idx2) + "]")
for key in all_validations[idx2].__iter__():
# if str(key) == 'out_temp_omm_min':
# key = 'out_temp_omm_min'
if str(key).endswith("time") and str(key).endswith(
"_time") is False:
new_val_val = {
"k": str(key).replace("time", "_time"),
"v": all_validations[idx2][key],
"idx2": idx2,
"idx": idx,
}
val_to_add_val.append(new_val_val)
# fix XX_sum -> XX_s
if str(key).endswith(
"_sum") and str(key).endswith("_s") is False:
new_val_val = {
"k": str(key).replace("_sum", "_s"),
"v": all_validations[idx2][key],
"idx2": idx2,
"idx": idx,
}
val_to_add_val.append(new_val_val)
if str(key).endswith("_max") or str(key).endswith(
"_min"):
if all_validations[idx2].__contains__(
key + "time") is True and all_validations[
idx2].__contains__(key +
"_time") is False:
new_val_val = {
"k": key + '_time',
"v": all_validations[idx2][key],
"idx2": idx2,
"idx": idx,
}
val_to_add_val.append(new_val_val)
if all_validations[idx2].__contains__(
key + "_time") is False:
new_val_val = {
"k": key + "_time",
"v": j["data"][idx]["stop_dat"],
"idx2": idx2,
"idx": idx,
}
val_to_add_val.append(new_val_val)
idx2 += 1
idx += 1
# add missing key/value
for my_val in val_to_add:
my_data = j["data"][my_val["idx"]]
if my_val.get("k2") is None:
my_data[my_val["k"]] = my_val["v"]
else:
my_data[my_val["k2"]][my_val["k"]] = my_val["v"]
for my_val in val_to_add_agg:
my_aggregations = j["data"][my_val["idx"]]["aggregations"]
my_aggregations[my_val["idx2"]][my_val["k"]] = my_val["v"]
for my_val in val_to_add_val:
my_validation = j["data"][my_val["idx"]]["validation"]
my_validation[my_val["idx2"]][my_val["k"]] = my_val["v"]
return None
except Exception as e:
if e.__dict__.__len__() == 0 or "done" not in e.__dict__:
exception_type, exception_object, exception_traceback = sys.exc_info(
)
exception_info = e.__repr__()
filename = exception_traceback.tb_frame.f_code.co_filename
funcname = exception_traceback.tb_frame.f_code.co_name
line_number = exception_traceback.tb_lineno
e.info = {
"i": str(exception_info),
"n": funcname,
"f": filename,
"l": line_number,
}
e.done = True
raise e