def build_diff(args): (k, time_slots, conf_lev, set_val, set_name, num_type) = args log.info(set_name) try: diff_mean = get_diff(set_val, time_slots, num_type, conf_lev) if num_type == FLOAT_TYPE: #norm_diff_mean,output_status=normalize_data(diff_mean[:,0]) norm_diff_mean,output_status=normalize_data(diff_mean) elif num_type == INT_TYPE: #num_discrete_vals=len(set(list(diff_mean[:,0]))) num_discrete_vals=len(set(list(diff_mean))) log.info('num_discrete_vals :' + str(num_discrete_vals)) if num_discrete_vals>1: output_status = 0 norm_diff_mean = diff_mean else: output_status = -1 norm_diff_mean = list(set(diff_mean)) #norm_diff_mean=list(set(diff_mean[:,0])) else: pass except Exception as e: log.error(traceback.print_exc()) log.error('Error in processing data feature, excluded from analysis ' + str(e)) output_status = -1 norm_diff_mean = None return (k,[output_status, norm_diff_mean]) return (k, [output_status, norm_diff_mean])
def upload_compact(request): """ 上传合同模板 :param request: :return: """ if request.method == 'POST': compact = request.FILES.get('compact', None) if not compact: log.error('合同模板上传失败:没有上传文件') msg = '请先选择要上传的文件!' status = 400 else: try: with open(BASE_DIR + '/static/media/compact/' + compact.name, 'wb+') as f: for chunk in compact.chunks(): f.write(chunk) msg = '上传成功!' status = 200 except Exception as e: log.error('合同模板上传失败:' + str(e)) msg = '上传失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def rotate_pic(request): """ 旋转图片 :param request: :return: """ pic_url = request.GET.get('pic_url', '') if any((not pic_url, len(pic_url) == 0)): log.error('旋转图片失败:无效的文件名') msg = '操作失败!' status = 400 else: try: pic_name = pic_url.split('/')[-1] im = Image.open( os.path.join(BASE_DIR, 'static/media/pic/') + pic_name) out = im.transpose(Image.ROTATE_270) out.save(os.path.join(BASE_DIR, 'static/media/pic/') + pic_name) msg = '操作成功!' status = 200 except Exception as e: log.error('旋转图片失败:' + str(e)) msg = '操作失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def register(request): """ 会员申请 :param request: :return: """ if request.method == 'POST': name = request.POST.get('name', '') phone = request.POST.get('phone', '') if any((not name, not phone, len(name) == 0, len(phone) == 0)): log.error('申请会员失败:名字或电话号码不能为空') msg = '用户名或手机号不能为空' status = 400 else: ret = re.match(r"^1[3456789]\d{9}$", phone) if not ret: msg = '手机号格式错误' status = 400 else: try: if Members.objects.filter(phone=phone).exists(): return JsonResponse({ 'status': 400, 'msg': '该手机号已绑定会员账号,请更换手机号或直接登录!' }) Members.objects.create(name=name, phone=phone, auth_str=(make_pwd(phone[-6:]))) status = 200 msg = '申请成功,您的初始密码为手机号后6位' except Exception as e: log.error('申请会员失败:' + str(e)) status = 400 msg = '申请失败!' return JsonResponse({'status': status, 'msg': msg})
def get_mem_materials(request): """ 获取用户上传的图片 :param request: :return: """ mem_id = request.GET.get('id', '') if any((not mem_id, len(mem_id) == 0)): log.error('获取用户上传的图片失败:无效的用户id') msg = '获取用户上传图片失败!' status = 400 else: try: materials = MemberMaterials.objects.filter(mem_id=mem_id) pic_list = [] for material in materials: pic_list.append( os.path.join(BASE_URL, 's/media/pic/') + material.materials_name) return JsonResponse({'status': 200, 'pic_list': pic_list}) except Exception as e: log.error('获取用户上传的图片失败:' + str(e)) msg = '获取用户上传图片失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def edit_addr(request): """ 修改地址 :param request: :return: """ if request.method == 'POST': addr_id = request.POST.get('id', '') area = request.POST.get('area', '') addr = request.POST.get('addr', '') print(addr_id, type(area), type(addr)) if any((not addr_id, not area, not addr, len(addr_id) == 0, len(area) == 0, len(addr) == 0)): msg = '区域或详细地址不能为空!' status = 400 else: try: area_addr = AreaAddr.objects.get(pk=addr_id) area_addr.area, area_addr.addr = area, addr area_addr.save() msg = '地址修改成功!' status = 200 except Exception as e: log.error('地址修改失败:' + str(e)) msg = '地址修改失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def get_diff(set_val,time_slots,num_type,conf_lev): time_slots_utc = dtime_to_unix(time_slots) TIMELET_INV_seconds = (time_slots[1]-time_slots[0]).seconds diff_mean = list() for r, utc_t in enumerate(time_slots_utc): utc_t_s = utc_t utc_t_e = utc_t + TIMELET_INV_seconds idx = np.nonzero((set_val[0] >= utc_t_s) & (set_val[0] < utc_t_e))[0] if len(idx) < 2: diff_val = np.inf else: temp_val = abs(np.diff(set_val[1][idx])) upper_val = np.sort(temp_val)[int(np.floor(len(temp_val)*conf_lev)):] if len(upper_val) == 0: diff_val = np.inf else: if num_type == FLOAT_TYPE: diff_val = np.mean(upper_val) elif num_type == INT_TYPE: diff_val = int(stats.mode(upper_val)[0]) else: log.error('Sample type must either INT or FLOAT type') raise NameError('Sample type must either INT or FLOAT type') #diff_val=max(abs(diff(set_val[1][idx]))) #sort(abs(diff(set_val[1][idx])))[::-1] diff_mean.append(diff_val) #diff_mean=np.array(diff_mean)[:,np.newaxis] diff_mean = np.array(diff_mean) return diff_mean
def apply_for_qualification(request): """ 申请资质 :param request: :return: """ if request.method == 'POST': mem_id = request.POST.get('mem_id', '') addr_id = request.POST.get('addr_id', '') if any((not mem_id, not addr_id, len(mem_id) == 0, len(addr_id) == 0)): log.error('申请资质失败:无效的参数') msg = '申请失败!' status = 400 else: try: member = Members.objects.get(pk=mem_id) if member.qualification_code == 1: return JsonResponse({ 'status': 400, 'msg': '您已申请资质,请勿重复申请!' }) member.area_id = addr_id member.qualification_code = 1 member.save() msg = '申请成功!' status = 200 except Exception as e: log.error('申请资质失败:' + str(e)) msg = '申请失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def get_area(request): """ 获取所有地址 :param request: :return: """ try: areas = AreaAddr.objects.all() area_list = [] for area in areas: area_list.append({ 'id': area.id, 'area': area.area, 'addr': area.addr }) return JsonResponse({ 'status': 200, 'msg': '操作成功!', 'data': { 'area_list': area_list } }) except Exception as e: log.error('获取地址失败:' + str(e)) return JsonResponse({'status': 400, 'msg': '操作失败!'})
def change_mem_status(request): """ 修改用户审批状态 :param request: :return: """ if request.method == 'POST': mem_id = request.POST.get('id', '') code = request.POST.get('code', '') if any((not mem_id, not code, len(mem_id) == 0, len(code) == 0)): log.error('修改客户资质失败:无效的用户id') msg = '操作失败!' status = 400 else: try: member = Members.objects.get(pk=mem_id) member.qualification_code = code member.save() msg = '操作成功!' status = 200 except Exception as e: log.error('修改客户资质失败:' + str(e)) msg = '操作失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def save_diff_data_summary_json(bldg_key, sensor_names_hash, diff_out): from log_util import log import traceback try: XDIFF_Sensor_STATE = diff_out['diffdata_state_mat'] #XDIFF_Weather_STATE = diff_out['diffdata_weather_mat'] #XDIFF_Time_STATE = diff_out['diffdata_time_mat'] #Xdiff_Time = diff_out['diff_time_slot'] #diffdata_zvar = diff_out['diffdata_zvar'] diffdata_exemplar = dict() for de, cl in diff_out['diffdata_exemplar'].iteritems(): kuid = sensor_names_hash[de] clist = list() for cn in cl: vuid = sensor_names_hash[cn] clist.append(vuid) diffdata_exemplar.update({kuid: clist}) XDIFF_Sensor_NAMES = list() for dn in diff_out['sensor_names']: uid = sensor_names_hash[dn] XDIFF_Sensor_NAMES.append(uid) #X_Weather_NAMES = diff_out['weather_names'] #X_Time_NAMES = diff_out['time_names'] num_cols = XDIFF_Sensor_STATE.shape[1] x_states = list() for c in xrange(0, num_cols): x_states.append(XDIFF_Sensor_STATE[:, c].tolist()) x_sensor_diff = {"sensor-names": XDIFF_Sensor_NAMES ,"sensor-exemplar": diffdata_exemplar ,"sensor-state": x_states} with open(JSON_DIR + bldg_key.lower() + "_sensor_feature_diff.json", 'w') as f: f.write(json.dumps(x_sensor_diff)) except Exception as e: log.error(traceback.print_exc()) log.error(str(e))
def login(request): """ 登录页 :param request: :return: """ if request.method == 'POST': # 获取用户名和口令 name = request.POST.get('username', '') pwd = request.POST.get('password', '') if any((not name, not pwd, len(name) == 0, len(pwd) == 0)): msg = '用户名或口令不能为空!' status = 400 else: try: ret = Members.objects.filter(name=name, auth_str=make_pwd(pwd)) if ret.exists(): login_user = ret.first() if login_user.activate == 1: # 将登陆的用户信息存在session中 request.session['login_mem_user'] = { 'id': login_user.id, 'name': login_user.name } return JsonResponse({ 'status': 200, 'msg': '登录成功!', 'data': { 'id': login_user.id } }) else: msg = '您的会员申请尚未通过,请与后台管理员联系!' status = 400 else: msg = '用户名或密码错误' status = 400 except Exception as e: log.error('用户登录失败:' + str(e)) msg = '登录失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def download_compact(request): """ 下载合同模板 :param request: :return: """ file_name = request.GET.get('file_name', '') if any((not file_name, len(file_name) == 0)): log.error('下载合同模板失败:无效的文件名') try: f = open(BASE_DIR + '/static/media/compact/' + file_name, 'rb') response = FileResponse(f) response['Content-Type'] = 'application/octet-stream' response[ 'Content-Disposition'] = f'attachment;filename="{urllib.parse.quote(file_name)}"' return response except Exception as e: log.error('下载合同模板失败:' + str(e))
def get_compact(request): """ 获取所有合同模板 :param request: :return: """ try: compact_list = os.listdir(BASE_DIR + '/static/media/compact') return JsonResponse({ 'status': 200, 'msg': '获取成功!', 'data': { 'compact_list': compact_list } }) except Exception as e: log.error('获取模板文件列表失败:' + str(e)) return JsonResponse({'status': 400, 'msg': '获取失败!'})
def pp_verify_sensor_data_format(tup): (key, data_list, time_slots, q) = tup log.info(' checking ' + key + '...') try: for i, samples in enumerate(data_list): for j, each_sample in enumerate(samples): if each_sample == []: q.put([key, i, j]) log.info(str(each_sample) + ' at ' + str(time_slots[i]) + ' in ' + str(key)) elif not isinstance(each_sample, int) and not isinstance(each_sample, float): q.put([key, i, j]) log.info(str(each_sample) + ' at ' + str(time_slots[i]) + ' in ' + str(key)) except Exception as e: log.error(str(e))
def pp_verify_sensor_data_format(tup): (key, data_list, time_slots, q) = tup log.info(' checking ' + key + '...') try: for i, samples in enumerate(data_list): for j, each_sample in enumerate(samples): if each_sample == []: q.put([key, i, j]) log.info(str(each_sample) + ' at ' + str(time_slots[i]) + ' in ' + str(key)) elif not isinstance(each_sample, int) and not isinstance(each_sample, float): q.put([key, i, j]) log.info(str(each_sample) + ' at ' + str(time_slots[i]) + ' in ' + str(key)) except Exception as e: log.error(traceback.print_exc()) log.error(str(e))
def save_avg_data_summary_json(bldg_key, sensor_names_hash, avg_out): from log_util import log import traceback try: X_Sensor_STATE = avg_out['avgdata_state_mat'] #X_Weather_STATE = avg_out['avgdata_weather_mat'] avgdata_exemplar = dict() for ae, cl in avg_out['avgdata_exemplar'].iteritems(): kuid = sensor_names_hash[ae] clist = list() for cn in cl: vuid = sensor_names_hash[cn] clist.append(vuid) avgdata_exemplar.update({kuid: clist}) #X_Sensor_NAMES = avg_out['sensor_names'] #X_Weather_NAMES = avg_out['weather_names'] X_Sensor_NAMES = list() for an in avg_out['sensor_names']: uid = sensor_names_hash[an] X_Sensor_NAMES.append(uid) num_cols = X_Sensor_STATE.shape[1] x_states = list() for c in xrange(0, num_cols): x_states.append(X_Sensor_STATE[:, c].tolist()) x_sensor_avg = {"sensor-names": X_Sensor_NAMES ,"sensor-exemplar": avgdata_exemplar ,"sensor-state": x_states} with open(JSON_DIR + bldg_key.lower() + "_sensor_feature_avg.json", 'w') as f: f.write(json.dumps(x_sensor_avg)) except Exception as e: log.error(traceback.print_exc()) log.error(str(e))
def save_processed_json(sensor_names_hash, ds_out): from log_util import log import traceback try: diff_name_list = ds_out['diffdata_dict']['build_diff_matrix_out'].Xdiff_Names avg_ts, avg_val = ts_npval(ds_out['avgdata_dict']['build_feature_matrix_out'].X_Time, ds_out['avgdata_dict']['build_feature_matrix_out'].X_Feature) diff_ts, diff_val = ts_npval(ds_out['diffdata_dict']['build_diff_matrix_out'].Xdiff_Time, ds_out['diffdata_dict']['build_diff_matrix_out'].Xdiff_Mat) ts_indices = list() ts_list = None if avg_ts.shape[0] < diff_ts.shape[0]: ts_indices = [np.nonzero(diff_ts == ts)[0] for ts in avg_ts] ts_list = avg_ts.tolist() else: ts_indices = [np.nonzero(avg_ts == ts)[0] for ts in diff_ts] ts_list = diff_ts.tolist() for i, name in enumerate(diff_name_list): avg = avg_val[:, i].tolist() if len(ts_indices) < avg_val[:, i].shape[0]: avg = avg_val[:, i][ts_indices][:, 0].tolist() diff = diff_val[:, i].tolist() if len(ts_indices) < diff_val[:, i].shape[0]: diff = diff_val[:, i][ts_indices][:, 0].tolist() json_out = list() for t_idx in xrange(0, len(ts_indices)): json_out.append([ts_list[t_idx], avg[t_idx], diff[t_idx]]) uid = sensor_names_hash[name] with open(JSON_DIR + "preproc-" + uid + ".json", 'w') as f: f.write(json.dumps(json_out)) except Exception as e: log.error(traceback.print_exc()) log.error(str(e))
def delete_compact(request): """ 删除合同模板 :param request: :return: """ file_name = request.GET.get('file_name', '') if any((not file_name, len(file_name) == 0)): log.error('合同模板删除失败:无效的文件名!') msg = '删除失败!' status = 400 else: try: os.remove(BASE_DIR + '/static/media/compact/' + file_name) msg = '删除成功!' status = 200 except Exception as e: log.error('合同模板删除失败:' + str(e)) msg = '删除失败!' status = 200 return JsonResponse({'status': status, 'msg': msg})
def compute_joint_prob(data_mat, state_idx_set, state_val_set): num_samples = data_mat.shape[0] num_states = data_mat.shape[1] if len(state_idx_set) != len(state_val_set): log.error('the length of state_set and state_val must be same') raise NameError('the length of state_set and state_val must be same') joint_idx = set(range(num_samples)) for k, state_idx in enumerate(state_idx_set): samples = data_mat[:, state_idx] sub_joint_idx = set([]) for state_val in state_val_set[k]: sub_joint_idx = sub_joint_idx | set(np.nonzero(samples == state_val)[0]) joint_idx = joint_idx & sub_joint_idx joint_prob = len(joint_idx)/num_samples if num_samples == 0: return 0 else: return joint_prob
def change_pwd(request): """ 修改密码 :param request: :return: """ if request.method == 'POST': mem_id = request.POST.get('mem_id', '') old_pwd = request.POST.get('old_pwd', '') new_pwd1 = request.POST.get('new_pwd1', '') new_pwd2 = request.POST.get('new_pwd2', '') if any((not mem_id, not old_pwd, not new_pwd1, not new_pwd2, len(mem_id) == 0, len(new_pwd1) == 0, len(new_pwd2) == 0)): log.error('密码修改失败:无效的参数') msg = '操作失败!' status = 400 else: if new_pwd1 != new_pwd2: log.error('密码修改失败:两次密码输入不一致!') msg = '两次输入的密码不一致!' status = 400 else: try: member = Members.objects.get(pk=mem_id) member.auth_str = make_pwd(new_pwd1) member.save() msg = '操作成功!' status = 200 except Exception as e: log.error('密码修改失败:' + str(e)) msg = '操作失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def get_qualification(request): """ 获取用户资质申请状态 :param request: :return: """ mem_id = request.GET.get('mem_id', '') if any((not mem_id, len(mem_id) == 0)): log.error('获取用户资质失败:无效的参数') else: try: member = Members.objects.get(pk=mem_id) return JsonResponse({ 'status': 200, 'msg': '查询成功!', 'data': { 'status': member.qualification.name } }) except Exception as e: log.error('获取用户资质失败:' + str(e)) return JsonResponse({'status': 400, 'msg': '查询失败!'})
def upload_pic(request): """ 上传申请资质照片 :param request: :return: """ if request.method == 'POST': mem_id = request.POST.get('mem_id', None) pic = request.FILES.get('pic', None) if any((not pic, not mem_id, len(mem_id) == 0)): log.error('照片上传失败:无效的参数') msg = '上传失败!' status = 400 else: try: member_material = MemberMaterials.objects.create( mem_id=mem_id, materials_name='{}_{}'.format(mem_id, pic.name)) with open( BASE_DIR + '/static/media/pic/{}_{}'.format(mem_id, pic.name), 'wb+') as f: for chunk in pic.chunks(): f.write(chunk) member_material.save() return JsonResponse({ 'status': 200, 'msg': '上传成功!', 'data': { 'pic_url': '/s/media/pic/{}_{}'.format(mem_id, pic.name) } }) except Exception as e: log.error('照片上传失败:' + str(e)) msg = '上传失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def add_addr(request): """ 添加地址 :param request: :return: """ if request.method == 'POST': area = request.POST.get('area', '') addr = request.POST.get('addr', '') if any((not area, not addr, len(area) == 0, len(addr) == 0)): msg = '区域或详细地址不能为空!' status = 400 else: try: AreaAddr.objects.create(area=area, addr=addr) msg = '添加成功' status = 200 except Exception as e: log.error('地址添加失败:' + str(e)) msg = '地址添加失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def delete_addr(request): """ 删除地址 :param request: :return: """ addr_id = request.GET.get('id', '') if any((not addr_id, len(addr_id) == 0)): log.error('地址删除失败:无效的id') msg = '删除失败!' status = 400 else: try: addr = AreaAddr.objects.get(pk=addr_id) addr.delete() msg = '删除成功!' status = 200 except Exception as e: log.error('地址删除失败:' + str(e)) msg = '删除失败!' status = 400 return JsonResponse({'status': status, 'msg': msg})
def compute_joint_prob(data_mat, state_idx_set, state_val_set): num_samples = data_mat.shape[0] num_states = data_mat.shape[1] if len(state_idx_set) != len(state_val_set): log.error('the length of state_set and state_val must be same') raise NameError('the length of state_set and state_val must be same') joint_idx = set(range(num_samples)) for k, state_idx in enumerate(state_idx_set): samples = data_mat[:, state_idx] sub_joint_idx = set([]) for state_val in state_val_set[k]: sub_joint_idx = sub_joint_idx | set( np.nonzero(samples == state_val)[0]) joint_idx = joint_idx & sub_joint_idx joint_prob = len(joint_idx) / num_samples if num_samples == 0: return 0 else: return joint_prob
def all_mem_apply_for(request): """ 获取所有申请资质的客户 :param request: :return: """ try: members = Members.objects.filter(qualification_code=1).order_by('-id') paginator = Paginator(members, 8, 0) page_num = request.GET.get('page_num', '') if page_num: if int(page_num) <= paginator.num_pages: page = paginator.page(page_num) else: page = paginator.page(paginator.num_pages) else: page = paginator.page(1) return render(request, 'sys_user/mem_apply_for_mgr.html', { 'page': page, 'paginator': paginator }) except Exception as e: log.error('查询所有资质待审核客户失败:' + str(e))
def get_initiation_mem(request): """ 获取所有申请入会的会员 :param request: :return: """ try: members = Members.objects.filter(activate=0).order_by('-id') paginator = Paginator(members, 8, 0) page_num = request.GET.get('page_num', '') if page_num: if int(page_num) <= paginator.num_pages: page = paginator.page(page_num) else: page = paginator.page(paginator.num_pages) else: page = paginator.page(1) return render(request, 'sys_user/mem_join_mgr.html', { 'page': page, 'paginator': paginator }) except Exception as e: log.error('获取所有申请入会会员失败:' + str(e))
def compact_mgr(request): """ 获取所用合同模板 :param request: :return: """ try: compact_list = os.listdir(BASE_DIR + '/static/media/compact') paginator = Paginator(compact_list, 8, 0) page_num = request.GET.get('page_num', '') if page_num: if int(page_num) <= paginator.num_pages: page = paginator.page(page_num) else: page = paginator.page(paginator.num_pages) else: page = paginator.page(1) return render(request, 'sys_user/compact_mgr.html', { 'page': page, 'paginator': paginator }) except Exception as e: log.error('获取模板文件列表失败:' + str(e))
def search_mem_qualification(request): """ 查询客户资质 :param request: :return: """ key = request.GET.get('key', '') members = [] if key: try: areas = AreaAddr.objects.filter( Q(area__contains=key) | Q(addr__contains=key)) areas_id = [area.id for area in areas] members = Members.objects.filter(Q(name__contains=key) | Q(phone=key) | Q(area_id__in=areas_id)) \ .order_by('-id') print('查询结果', members) except Exception as e: log.error('查询客户资质失败:' + str(e)) else: try: members = Members.objects.all().order_by('-id') except Exception as e: log.error('查询客户资质失败:' + str(e)) paginator = Paginator(members, 8, 0) page_num = request.GET.get('page_num', '') if page_num: if int(page_num) <= paginator.num_pages: page = paginator.page(page_num) else: page = paginator.page(paginator.num_pages) else: page = paginator.page(1) return render(request, 'sys_user/mem_qualification_search.html', { 'page': page, 'paginator': paginator, 'key': key })
def to_area_mgr(request): """ 查询地址 :param request: :return: """ try: areas = AreaAddr.objects.all().order_by('id') paginator = Paginator(areas, 8, 0) page_num = request.GET.get('page_num', '') if page_num: if int(page_num) <= paginator.num_pages: page = paginator.page(page_num) else: page = paginator.page(paginator.num_pages) else: page = paginator.page(1) return render(request, 'sys_user/area_mgr.html', { 'page': page, 'paginator': paginator }) except Exception as e: log.error('地址查询失败:' + str(e))
def activate_member(request): """ 批准用户会员申请 :param request: :return: """ mem_id = request.GET.get('id', '') code = request.GET.get('code', '') if any((not mem_id, not code, len(mem_id) == 0, len(code) == 0)): log.error('会员激活失败:参数不能为空!') msg = '操作失败!' status = 400 else: try: member = Members.objects.get(pk=mem_id) member.activate = code member.save() msg = '操作成功!' status = 200 except Exception as e: log.error('会员激活失败:' + str(e)) msg = '操作失败!' status = 200 return JsonResponse({'status': status, 'msg': msg})
def build_feature_matrix(data_dict, sensor_list, weather_list, time_slots, interpolation=1, max_num_succ_idx_for_itpl=4): data_used = sensor_list + weather_list log.info('Build data feature matrix now.....') if interpolation == 1: log.info('Missing samples will be interpolated upto ' + str(max_num_succ_idx_for_itpl) + 'successive time slots') else: log.info('All time slots with any missing sample will be removed without interpolatoin ') num_of_data = len(data_used) num_of_samples = len(time_slots) # Declare as 2-d list for exception. X = list() INT_type_list = list() FLOAT_type_list = list() input_names = list() weather_type_idx = list() sensor_type_idx = list() INT_type_idx = list() FLOAT_type_idx = list() zero_var_list = list() zero_var_val = list() # whose variance is zero, hence carry no information, # Constrcut X matrix by summerizing hourly samples for j, key in enumerate(data_used): log.info('-' * 40) log.info('building for ' + str(key)) try: num_type = check_data_type(data_dict[key][2][1]) # Avg. value feature x_temp = get_feature(data_dict[key][1], num_type) non_inf_idx = np.nonzero(x_temp < np.inf)[0] #if non_inf_idx <len(time_slots):measurement_point_set # Outlier removal, different parameters for sensors and weather data if len(sensor_list) <= j: # weather data is_weather_data = True outlier_idx = outlier_detect(x_temp[non_inf_idx], 5, 10) else: is_weather_data = False outlier_idx = outlier_detect(x_temp[non_inf_idx], 1, 20) if len(outlier_idx) > 0: log.info('outlier samples are detected: outlier_idx:' + str(outlier_idx)) x_temp[non_inf_idx[outlier_idx]] = np.inf # interplolation data, use nearest for int type, use linear for float type if interpolation == 1: x_temp = interploate_data(x_temp, num_type, max_num_succ_idx_for_itpl) norm_data_vec, output_status = normalize_data(x_temp[:, 0]) if len(np.nonzero(norm_data_vec == np.inf)[0]) > num_of_samples/5: raise except Exception as e: log.error(traceback.print_exc()) log.error(' Error in processing data feature, excluded from analysis ' + str(e)) output_status = -1 norm_data_vec = None if output_status == -1: zero_var_list.append(key) zero_var_val.append(norm_data_vec) log.info('too small variance for float type, added to zero var list') else: input_names.append(key) log.info(str(j)+'th sensor update') if (num_type == FLOAT_TYPE) and (is_weather_data == False): X.append(norm_data_vec) FLOAT_type_idx.append(len(X)-1) FLOAT_type_list.append(key) elif (num_type == INT_TYPE) or (is_weather_data == True): X.append(x_temp[:, 0]) INT_type_idx.append(len(X)-1) INT_type_list.append(key) else: log.error('Sample type must either INT or FLOAT type') raise NameError('Sample type must either INT or FLOAT type') if key in weather_list: weather_type_idx.append(len(X)-1) elif key in sensor_list: sensor_type_idx.append(len(X)-1) else: log.error('Sample type must either Weather or Sensor type') raise NameError('Sample type must either Weather or Sensor type') # Linear Interpolate X = np.array(X).T if X.shape[0] != num_of_samples: log.error('The numeber of rows in feature matrix and the number of the time slots are different ') raise NameError('The numeber of rows in feature matrix and the number of the time slots are different ') if X.shape[1]+len(zero_var_list) != num_of_data: log.error('The sume of the numeber of column in feature matrix and the number of zero var column are different from the number of input measurements ') raise NameError('The sume of the numeber of column in feature matrix and the number of zero var column are different from the number of input measurements ') deleted_timeslot_idx=[] log.info('-' * 20) log.info('removing time slots having no sample...') inf_idx_set = [] for col_vec in X.T: inf_idx = np.nonzero(col_vec ==np.infty)[0] inf_idx_set = np.r_[inf_idx_set, inf_idx] inf_col_idx = list(set(list(inf_idx_set))) deleted_timeslot_idx = np.array([int(x) for x in inf_col_idx]) log.info('time slots ' + str(deleted_timeslot_idx) + ' removed...') log.info('-' * 20) X = np.delete(X, deleted_timeslot_idx, axis=0) new_time_slot = np.delete(time_slots, deleted_timeslot_idx) # Checking whether it has any ill entry value verify_data_mat(X) return X, new_time_slot, input_names, zero_var_list, zero_var_val, INT_type_list, INT_type_idx, FLOAT_type_list, FLOAT_type_idx, weather_type_idx, sensor_type_idx
def plotting_bldg_bn(bldg): plt.ioff() log.info('Getting anal_out from ' + bldg.bldg_tag) try: for sig_tag, anal_out in bldg.anal_out.iteritems(): for bn_prob in anal_out: p_name = bn_prob['p_name'] try: fig_name = 'BN for Sensors ' + p_name plt.figure(fig_name, figsize=(30.0, 30.0)) col_name = bn_prob['s_labels'] rbn.nx_plot(bn_prob['s_hc'], col_name, graph_layout='spring', node_text_size=30) plt.savefig(FIG_DIR + bldg.bldg_tag + '_' + p_name + '_' + sig_tag + '_bn_sensors' + get_pngid() + '.png', bbox_inches='tight') plt.close() except Exception as e: log.error(traceback.print_exc()) log.error('error in ' + fig_name + ' ' + str(e)) pass try: fig_name = 'BN for Time ' + p_name plt.figure(fig_name, figsize=(30.0,30.0)) rbn.nx_plot(bn_prob['t_hc'], bn_prob['t_labels'], graph_layout='spring', node_text_size=30) plt.savefig(FIG_DIR + bldg.bldg_tag + '_' + p_name + '_' + sig_tag + '_bn_time' + get_pngid() + '.png', bbox_inches='tight') plt.close() except Exception as e: log.error(traceback.print_exc()) log.error('error in ' + fig_name + ' ' + str(e)) pass try: fig_name = 'BN for Weather ' + p_name plt.figure(fig_name, figsize=(30.0,30.0)) rbn.nx_plot(bn_prob['w_hc'], bn_prob['w_labels'], graph_layout='spring', node_text_size=30) plt.savefig(FIG_DIR + bldg.bldg_tag + '_' + p_name + '_' + sig_tag + '_bn_weather' + get_pngid() +'.png', bbox_inches='tight') plt.close() except Exception as e: log.error(traceback.print_exc()) log.error('error in ' + fig_name + ' ' + str(e)) pass try: fig_name = 'BN for Sensor-Time-Weather ' + p_name plt.figure(fig_name, figsize=(30.0,30.0)) rbn.nx_plot(bn_prob['all_hc'], bn_prob['all_labels'], graph_layout='spring', node_text_size=30) plt.savefig(FIG_DIR + bldg.bldg_tag + '_' + p_name + '_' + sig_tag + '_bn_sensor_time_weather' + get_pngid() + '.png', bbox_inches='tight') plt.close() except Exception as e: log.error(traceback.print_exc()) log.error('error in ' + fig_name + ' ' + str(e)) pass try: fig_name = 'BN PEAK LH Analysis for Sensor-Time-Weather ' + p_name plt.figure(fig_name, figsize=(30.0, 30.0)) plt.subplot(2, 1, 1) plt.plot(bn_prob['all_cause_symbol_xtick'], bn_prob['high_peak_prob'], '-^') plt.plot(bn_prob['all_cause_symbol_xtick'], bn_prob['low_peak_prob'], '-.v') plt.ylabel('Likelihood', fontsize=20) plt.xticks(bn_prob['all_cause_symbol_xtick'], bn_prob['all_cause_symbol_xlabel'], rotation=270, fontsize=20) plt.tick_params(labelsize=20) plt.legend(('High Peak', 'Low Peak'), loc='center right', prop={'size':25}) plt.tick_params(labelsize=20) plt.grid() plt.ylim([-0.05,1.05]) plt.title('Likelihood of '+ str(remove_dot(p_name))+' given '+'\n'+str(remove_dot(bn_prob['all_cause_label'])), fontsize=20) plt.savefig(FIG_DIR + bldg.bldg_tag + '_' + p_name + '_' + sig_tag + '_LH_sensor_time_weather' + get_pngid() + '.png', bbox_inches='tight') plt.close() except Exception as e: log.error(traceback.print_exc()) log.error('error in ' + fig_name + ' ' + str(e)) pass except Exception as e: log.error(traceback.print_exc()) log.error(str(e)) pass plt.ion()
def plotting_bldg_lh(bldg, bldg_key=[], attr='sensor', num_picks=30): log.info('-' * 40) log.info('plotting lh for ' + attr) log.info('-' * 40) sig_tag_set = ['avg', 'diff'] plt.ioff() if not len(bldg_key): bldg_tag_set = [bldg.bldg_tag] else: bldg_tag_set = [bldg_key] for bldg_tag in bldg_tag_set: if bldg_tag == bldg.bldg_tag: log.info('-' * 40) log.info(bldg_tag + " is to be plotted...") log.info('-' * 40) for sig_tag in sig_tag_set: try: p_names = bldg.sigtags[sig_tag].p_names for pname in p_names: try: blank_idx = pname.index('.') pname = pname.replace('.', '_') except: pass optprob_set = None optstate_set = None for anal in bldg.analysis[sig_tag]: if anal.sensor_tag == pname: optprob_set = anal.attrs[attr].optprob_set optstate_set = anal.attrs[attr].optstate_set break s_names = bldg.sigtags[sig_tag].names[attr] num_picks = 30 sort_idx = np.argsort(optprob_set)[::-1] sort_lh = optprob_set[sort_idx[:num_picks]].T sort_state = optstate_set[sort_idx[:num_picks]].T x_label = list(np.array(s_names)[sort_idx[:num_picks]]) x_ticks = range(len(x_label)) plt.figure(figsize=(20.0, 15.0)) plt.subplot(2, 1, 1) plt.plot(sort_lh, '-*') plt.xticks(x_ticks, x_label, rotation=270, fontsize="small") if sig_tag == 'avg': plt.title('Most relavant ' + attr + ' attributes to the peak (demand) of '+ pname, fontsize=20) else: plt.title('Most relavant ' + attr + ' attributes to the peak variations of '+ pname, fontsize=20) plt.tick_params(labelsize='large') plt.ylim([-0.05, 1.05]) plt.ylabel('Likelihood (From 0 to 1)', fontsize=18) plt.savefig(FIG_DIR + bldg_tag + '_' + pname + '_' + attr + '_' + sig_tag + '_lh_sensors.png', bbox_inches='tight') plt.close() except Exception as e: log.error(traceback.print_exc()) log.error(str(e)) pass plt.close() plt.ion()
def data_summerization(bldg_key, data_dict, proc_avg=True, proc_diff=True, PARALLEL=False): time_slots = data_dict['time_slots'][:] conditions_dict = data_dict['Conditions_dict'].copy() events_dict = data_dict['Events_dict'].copy() sensor_list = data_dict['sensor_list'][:] weather_list = data_dict['weather_list'][:] weather_list_used = ['TemperatureC', 'Dew PointC', 'Humidity', 'Events', 'Conditions'] # data_used is the list of refernece name for all measurements from now on. data_used = sensor_list + weather_list_used # This is a global ID for data_used measurement data_used_idx = range(len(data_used)) sensor_idx = range(len(sensor_list)) weather_idx = range(len(sensor_list), len(data_used)) dsout = {'data_dict': data_dict} if proc_avg: log.info('-' * 40) log.info('processing avg.feature..') log.info('-' * 40) X_Feature, X_Time, X_names, X_zero_var_list, X_zero_var_val, X_int_type_list,\ X_int_type_idx, X_float_type_list, X_float_type_idx, X_weather_type_idx, X_sensor_type_idx = \ build_feature_matrix(data_dict, sensor_list, weather_list_used, time_slots, interpolation=1, max_num_succ_idx_for_itpl=int(len(time_slots)*0.05)) build_feature_matrix_out = \ {'X_Feature': X_Feature, 'X_Time': X_Time, 'X_names': X_names, 'X_zero_var_list': X_zero_var_list, 'X_zero_var_val': X_zero_var_val, 'X_int_type_list': X_int_type_list, 'X_int_type_idx': X_int_type_idx, 'X_float_type_list': X_float_type_list, 'X_float_type_idx': X_float_type_idx, 'X_weather_type_idx': X_weather_type_idx, 'X_sensor_type_idx': X_sensor_type_idx} build_feature_matrix_out = obj(build_feature_matrix_out) if len(X_names+X_zero_var_list) != len(data_used): log.error('Missing name is found in X_names or X_zero_var_list') raise NameError('Missing name is found in X_names or X_zero_var_list') else: zero_var_idx = [data_used.index(name_str) for name_str in X_zero_var_list] nzero_var_idx = list(set(data_used_idx)-set(zero_var_idx)) if X_Feature.shape[0] > 0: # From below all index are reference to X_Feature sf_idx = list(set(X_sensor_type_idx)&set(X_float_type_idx)) # Equivalent to np.array(data_used)[np.array(nzero_var_idx)[sf_idx]] sf_name = list(np.array(X_names)[sf_idx]) si_idx = list(set(X_sensor_type_idx)&set(X_int_type_idx)) si_name = list(np.array(X_names)[si_idx]) wf_idx = list(set(X_weather_type_idx)&set(X_float_type_idx)) wf_name = list(np.array(X_names)[wf_idx]) wi_idx = list(set(X_weather_type_idx)&set(X_int_type_idx)) wi_name = list(np.array(X_names)[wi_idx]) #Euclidian Distance Matrix of Floating type of data only wf+o float_idx = list(set(sf_idx)| set(wf_idx)) int_idx = list(set(si_idx)| set(wi_idx)) # Float Type Measurement Clustering X_Feature_sfe, sf_exemplars_dict, exemplars_, labels_ = \ cluster_measurement_points(X_Feature[:, sf_idx], sf_name, corr_bnd=[0.1, 0.9], alg='aff') sfe_idx = list(np.array(sf_idx)[exemplars_]) #plot_label(X_Feature,X_names,labels_,exemplars_,[4,5,6,7]) # InT Type Measurement Clustering X_Feature_sie, si_exemplars_dict, exemplars_, labels_ = \ cluster_measurement_points(X_Feature[:, si_idx], si_name, corr_bnd=[0.0, 0.9], alg='aff') sie_idx = list(np.array(si_idx)[exemplars_]) # sensor -float type sfe_state, sfe_corr_val = x_input_to_states(X_Feature_sfe, CORR_VAL_OUT=1) # sensor -integer type sie_state = X_Feature_sie # weather -float type wf_state, wf_corr_val = x_input_to_states(X_Feature[:, wf_idx], CORR_VAL_OUT=1) # weather -integer type wi_state = X_Feature[:, wi_idx] empty_states = np.array([[] for i in range(len(X_Time))]) if len(sfe_state) == 0: sfe_state = empty_states if len(sie_state) == 0: sie_state = empty_states if len(wf_state) ==0: wf_state = empty_states if len(wi_state) == 0: wi_state = empty_states # Exemplar sensor only X_Sensor_STATE = np.append(sfe_state,sie_state, axis=1) X_Sensor_STATE = X_Sensor_STATE.astype(int) X_Sensor_NAMES = list(np.array(X_names)[sfe_idx]) + list(np.array(X_names)[sie_idx]) X_Weather_STATE = np.append(wf_state,wi_state, axis=1) X_Weather_STATE = X_Weather_STATE.astype(int) X_Weather_NAMES = list(np.array(X_names)[wf_idx])+list(np.array(X_names)[wi_idx]) # months of a year,days of a week, and hours of a day # (Monday, Tuesday,Wendsday,Thursday,Saturday,Sunday) =(0,1,2,3,4,5,6) X_Time_STATE_temp = build_time_states(X_Time) X_Time_NAMES_temp = ['MTH', 'WD', 'HR'] X_Time_STATE = list() X_Time_NAMES = list() for xt_col, xt_name in zip(X_Time_STATE_temp.T,X_Time_NAMES_temp): if len(set(xt_col)) > 1: X_Time_STATE.append(xt_col) X_Time_NAMES.append(xt_name) X_Time_STATE = np.array(X_Time_STATE).T ################################################# # FORMATTED DATA FOR REGUALR EVENT ################################################# #DO_PROB_EST=1 ** Save this variables*** #avgdata_mat = np.hstack([X_Sensor_STATE,X_Weather_STATE,X_Time_STATE]) #avgdata_names = X_Sensor_NAMES+X_Weather_NAMES+X_Time_NAMES avgdata_exemplar = dict(sf_exemplars_dict.items()+si_exemplars_dict.items()) avgdata_zvar = X_zero_var_list avgdata_dict = dict() avgdata_dict.update({'build_feature_matrix_out': build_feature_matrix_out}) avgdata_dict.update({'avgdata_state_mat': X_Sensor_STATE}) avgdata_dict.update({'avgdata_weather_mat': X_Weather_STATE}) avgdata_dict.update({'avgdata_time_mat': X_Time_STATE}) avgdata_dict.update({'avg_time_slot': X_Time}) avgdata_dict.update({'avgdata_exemplar': avgdata_exemplar}) avgdata_dict.update({'avgdata_zvar': avgdata_zvar}) avgdata_dict.update({'sensor_names': X_Sensor_NAMES}) avgdata_dict.update({'weather_names': X_Weather_NAMES}) avgdata_dict.update({'time_names': X_Time_NAMES}) dsout.update({'avgdata_dict': avgdata_dict}) if proc_diff: log.info('-' * 40) log.info('processing diff.feature..') log.info('-' * 40) #################################### # Irregular Event Extraction #################################### # Interpolatoin with outlier removal, Here we exclude weather data from irregualr event analysis # since weather data noramlly show slow changes in time.so we dont expect in any meaningful diffs values measurement_point_set, num_type_set = interpolation_measurement(data_dict, sensor_list, err_rate=1, sgm_bnd=20) # Irregualr matrix Xdiff_Mat,\ Xdiff_Time,\ Xdiff_Names,\ Xdiff_zero_var_list,\ Xdiff_zero_var_val,\ Xdiff_int_type_list,\ Xdiff_int_type_idx,\ Xdiff_float_type_list,\ Xdiff_float_type_idx =\ build_diff_matrix(measurement_point_set, time_slots, num_type_set, sensor_list, PARALLEL=PARALLEL) build_diff_matrix_out = \ {'Xdiff_Mat':Xdiff_Mat, 'Xdiff_Time':Xdiff_Time, 'Xdiff_Names':Xdiff_Names, 'Xdiff_zero_var_list':Xdiff_zero_var_list, 'Xdiff_zero_var_val':Xdiff_zero_var_val, 'Xdiff_int_type_list':Xdiff_int_type_list, 'Xdiff_int_type_idx':Xdiff_int_type_idx, 'Xdiff_float_type_list':Xdiff_float_type_list, 'Xdiff_float_type_idx':Xdiff_float_type_idx} build_diff_matrix_out = obj(build_diff_matrix_out) if Xdiff_Mat.shape[0] > 0: #============================================================================== # Restructure diff_marix's and weather matix for the same common time slot #============================================================================== time_slots_array = np.sort(np.array(list(set(Xdiff_Time) & set(X_Time)))) # Extract subset of X_Weather_STATE removed_idx_list = list() for ridx, slot in enumerate(X_Time): slot_idx = np.where(time_slots_array==slot)[0] # slot not in common time slots if len(slot_idx) == 0: removed_idx_list.append(ridx) XDIFF_Weather_STATE = np.delete(X_Weather_STATE, removed_idx_list,axis=0) # Extract subset of Xdiff_Mat removed_idx_list = list() for ridx,slot in enumerate(Xdiff_Time): slot_idx = np.where(time_slots_array == slot)[0] # slot not in common time slots if len(slot_idx) == 0: removed_idx_list.append(ridx) Xdiff_Mat = np.delete(Xdiff_Mat, removed_idx_list, axis=0) # Update Xdiff_Time Xdiff_Time = time_slots_array XDIFF_Weather_STATE = np.array(XDIFF_Weather_STATE) # From below all index are reference to X_Feature xdiff_sf_idx = Xdiff_float_type_idx xdiff_sf_name = Xdiff_float_type_list xdiff_si_idx = Xdiff_int_type_idx xdiff_si_name = Xdiff_int_type_list # Float Type Measurement Clustering X_Diff_sfe, sf_diff_exemplars_dict, exemplars_, labels_ = \ cluster_measurement_points(Xdiff_Mat[:, xdiff_sf_idx], xdiff_sf_name, corr_bnd=[0.1, 0.9]) xdiff_sfe_idx = list(np.array(xdiff_sf_idx)[exemplars_]) # InT Type Measurement Clustering X_Diff_sie, si_diff_exemplars_dict, exemplars_, labels_ = \ cluster_measurement_points(Xdiff_Mat[:, xdiff_si_idx], xdiff_si_name, corr_bnd=[0.1, 0.9]) xdiff_sie_idx = list(np.array(xdiff_si_idx)[exemplars_]) # sensor -float type xdiff_sfe_state, xdiff_sfe_corr_val =\ x_input_to_states(X_Diff_sfe, CORR_VAL_OUT=1, PARALLEL=PARALLEL) # sensor -integer type xdiff_sie_state = X_Diff_sie empty_states = np.array([[] for i in range(len(Xdiff_Time))]) if len(xdiff_sfe_state) == 0: xdiff_sfe_state = empty_states if len(xdiff_sie_state) == 0: xdiff_sie_state = empty_states if len(wf_state) == 0: wf_state = empty_states if len(wi_state) == 0: wi_state = empty_states # Exemplar sensor only XDIFF_Sensor_STATE = np.append(xdiff_sfe_state,xdiff_sie_state, axis=1) XDIFF_Sensor_STATE = XDIFF_Sensor_STATE.astype(int) XDIFF_Sensor_NAMES = list(np.array(Xdiff_Names)[xdiff_sfe_idx])+list(np.array(Xdiff_Names)[xdiff_sie_idx]) # months of a year,days of a week, and hours of a day # (Monday, Tuesday,Wendsday,Thursday,Saturday,Sunday) =(0,1,2,3,4,5,6) XDIFF_Time_STATE_temp = build_time_states(Xdiff_Time) XDIFF_Time_NAMES_temp = ['MTH', 'WD', 'HR'] XDIFF_Time_STATE = list() XDIFF_Time_NAMES = list() for xt_col, xt_name in zip(XDIFF_Time_STATE_temp.T, XDIFF_Time_NAMES_temp): if len(set(xt_col)) > 1: XDIFF_Time_STATE.append(xt_col) XDIFF_Time_NAMES.append(xt_name) XDIFF_Time_STATE = np.array(XDIFF_Time_STATE).T ################################################# # FORMATTED DATA FOR IRREGUALR EVENT ################################################# log.info("FORMATTED DATA FOR IRREGUALR EVENT") #** Save this variables*** #diffdata_mat = np.hstack([XDIFF_Sensor_STATE,X_Weather_STATE,XDIFF_Time_STATE]) #diffdata_names = XDIFF_Sensor_NAMES+X_Weather_NAMES+XDIFF_Time_NAMES diffdata_exemplar = dict(sf_diff_exemplars_dict.items() + si_diff_exemplars_dict.items()) diffdata_zvar = Xdiff_zero_var_list diffdata_dict = dict() diffdata_dict.update({'build_diff_matrix_out': build_diff_matrix_out}) diffdata_dict.update({'diffdata_state_mat': XDIFF_Sensor_STATE}) diffdata_dict.update({'diffdata_weather_mat': XDIFF_Weather_STATE}) diffdata_dict.update({'diffdata_time_mat': XDIFF_Time_STATE}) diffdata_dict.update({'diff_time_slot': Xdiff_Time}) diffdata_dict.update({'diffdata_exemplar': diffdata_exemplar}) diffdata_dict.update({'diffdata_zvar': diffdata_zvar}) diffdata_dict.update({'sensor_names': XDIFF_Sensor_NAMES}) diffdata_dict.update({'weather_names': X_Weather_NAMES}) diffdata_dict.update({'time_names': X_Time_NAMES}) dsout.update({'diffdata_dict': diffdata_dict}) dsout.update({'bldg_key': remove_dot(bldg_key)}) return dsout
def run(self): from log_util import log try: while True: cmd = None try: cmd = self.cmd_q.get(block=True, timeout=0.1) except Exception as e: continue finally: if cmd: self.cmd_q.task_done() try: with open(META_DIR + "wip.json", 'w') as f: f.write(simplejson.dumps({"wip": 1})) cmdset = simplejson.loads(cmd) sensor_hash = cmdset['selected-nodes'] s_date = datetime.strptime(cmdset['start-date'], '%Y-%m-%d') e_date = datetime.strptime(cmdset['end-date'], '%Y-%m-%d') if not len(sensor_hash): log.critical("No sensor is selected!") else: log.info('****************************** Begining of DDEA *******************************') bldg_key = 'SODA' #exemplar by user #pname_key = '_POWER_' pname_key = 'POWER' s_epoch = int(time.mktime(s_date.timetuple())) e_epoch = int(time.mktime(e_date.timetuple())) time_inv = dt.timedelta(seconds=cmdset['time-interval']) log.info("Cleaning up old output...") mt.remove_all_files(FIG_DIR) mt.remove_all_files(JSON_DIR) mt.remove_all_files(PROC_OUT_DIR) log.info("start epoch : " + str(s_epoch) + " end epoch : " + str(e_epoch)) log.info(str(time_inv) + ' time slot interval is set for this data set !!!') log.info("BLDG_KEY : " + bldg_key + " PNAME_KEY : " + pname_key) log.info('*' * 80) log.info("Retrieve sensor data from quasar TSDB") sensor_names_hash = mt.sensor_name_uid_dict(bldg_key, sensor_hash) sensor_data = read_sensor_data(sensor_names_hash, s_epoch, e_epoch) if sensor_data and len(sensor_data): ddea_process(sensor_names_hash, sensor_data, s_epoch, e_epoch, time_inv, bldg_key, pname_key) else: log.critical("No sensor data available for time period and sensor selected!") log.info('******************************** End of DDEA **********************************') os.remove(META_DIR + "wip.json") cmd_lock.clear() log.info("execution-lock cleared") log.info('~' * 80) except Exception as e: os.remove(META_DIR + "wip.json") cmd_lock.clear() print e log.error(str(e)) except Exception as e: os.remove(META_DIR + "wip.json") cmd_lock.clear() print e log.error(str(e)) finally: sys.exit(0)
def data_summerization(bldg_key, data_dict, proc_avg=True, proc_diff=True, PARALLEL=False): time_slots = data_dict['time_slots'][:] conditions_dict = data_dict['Conditions_dict'].copy() events_dict = data_dict['Events_dict'].copy() sensor_list = data_dict['sensor_list'][:] weather_list = data_dict['weather_list'][:] weather_list_used = ['TemperatureC', 'Dew PointC', 'Humidity', 'Events', 'Conditions'] # data_used is the list of refernece name for all measurements from now on. data_used = sensor_list + weather_list_used # This is a global ID for data_used measurement data_used_idx = range(len(data_used)) sensor_idx = range(len(sensor_list)) weather_idx = range(len(sensor_list), len(data_used)) dsout = {'data_dict': data_dict} if proc_avg: log.info('-' * 40) log.info('processing avg.feature..') log.info('-' * 40) X_Feature, X_Time, X_names, X_zero_var_list, X_zero_var_val, X_int_type_list,\ X_int_type_idx, X_float_type_list, X_float_type_idx, X_weather_type_idx, X_sensor_type_idx = \ build_feature_matrix(data_dict, sensor_list, weather_list_used, time_slots, interpolation=1, max_num_succ_idx_for_itpl=int(len(time_slots)*0.05)) build_feature_matrix_out = \ {'X_Feature': X_Feature, 'X_Time': X_Time, 'X_names': X_names, 'X_zero_var_list': X_zero_var_list, 'X_zero_var_val': X_zero_var_val, 'X_int_type_list': X_int_type_list, 'X_int_type_idx': X_int_type_idx, 'X_float_type_list': X_float_type_list, 'X_float_type_idx': X_float_type_idx, 'X_weather_type_idx': X_weather_type_idx, 'X_sensor_type_idx': X_sensor_type_idx} build_feature_matrix_out = obj(build_feature_matrix_out) if len(X_names+X_zero_var_list) != len(data_used): log.error('Missing name is found in X_names or X_zero_var_list') raise NameError('Missing name is found in X_names or X_zero_var_list') else: zero_var_idx = [data_used.index(name_str) for name_str in X_zero_var_list] nzero_var_idx = list(set(data_used_idx)-set(zero_var_idx)) if X_Feature.shape[0] > 0: # From below all index are reference to X_Feature sf_idx = list(set(X_sensor_type_idx)&set(X_float_type_idx)) # Equivalent to np.array(data_used)[np.array(nzero_var_idx)[sf_idx]] sf_name = list(np.array(X_names)[sf_idx]) si_idx = list(set(X_sensor_type_idx)&set(X_int_type_idx)) si_name = list(np.array(X_names)[si_idx]) wf_idx = list(set(X_weather_type_idx)&set(X_float_type_idx)) wf_name = list(np.array(X_names)[wf_idx]) wi_idx = list(set(X_weather_type_idx)&set(X_int_type_idx)) wi_name = list(np.array(X_names)[wi_idx]) #Euclidian Distance Matrix of Floating type of data only wf+o float_idx = list(set(sf_idx)| set(wf_idx)) int_idx = list(set(si_idx)| set(wi_idx)) # Float Type Measurement Clustering X_Feature_sfe, sf_exemplars_dict, exemplars_, labels_ = \ cluster_measurement_points(X_Feature[:, sf_idx], sf_name, corr_bnd=[0.1, 0.9], alg='aff') sfe_idx = list(np.array(sf_idx)[exemplars_]) #plot_label(X_Feature,X_names,labels_,exemplars_,[4,5,6,7]) # InT Type Measurement Clustering X_Feature_sie, si_exemplars_dict, exemplars_, labels_ = \ cluster_measurement_points(X_Feature[:, si_idx], si_name, corr_bnd=[0.0, 0.9], alg='aff') sie_idx = list(np.array(si_idx)[exemplars_]) # sensor -float type sfe_state, sfe_corr_val = x_input_to_states(X_Feature_sfe, CORR_VAL_OUT=1) # sensor -integer type sie_state = X_Feature_sie # weather -float type wf_state, wf_corr_val = x_input_to_states(X_Feature[:, wf_idx], CORR_VAL_OUT=1) # weather -integer type wi_state = X_Feature[:, wi_idx] empty_states = np.array([[] for i in range(len(X_Time))]) if len(sfe_state) == 0: sfe_state = empty_states if len(sie_state) == 0: sie_state = empty_states if len(wf_state) ==0: wf_state = empty_states if len(wi_state) == 0: wi_state = empty_states # Exemplar sensor only X_Sensor_STATE = np.append(sfe_state,sie_state, axis=1) X_Sensor_STATE = X_Sensor_STATE.astype(int) X_Sensor_NAMES = list(np.array(X_names)[sfe_idx]) + list(np.array(X_names)[sie_idx]) X_Weather_STATE = np.append(wf_state,wi_state, axis=1) X_Weather_STATE = X_Weather_STATE.astype(int) X_Weather_NAMES = list(np.array(X_names)[wf_idx])+list(np.array(X_names)[wi_idx]) # months of a year,days of a week, and hours of a day # (Monday, Tuesday,Wendsday,Thursday,Saturday,Sunday) =(0,1,2,3,4,5,6) X_Time_STATE_temp = build_time_states(X_Time) X_Time_NAMES_temp = ['MTH', 'WD', 'HR'] X_Time_STATE = list() X_Time_NAMES = list() for xt_col, xt_name in zip(X_Time_STATE_temp.T,X_Time_NAMES_temp): if len(set(xt_col)) > 1: X_Time_STATE.append(xt_col) X_Time_NAMES.append(xt_name) X_Time_STATE = np.array(X_Time_STATE).T ################################################# # FORMATTED DATA FOR REGUALR EVENT ################################################# #DO_PROB_EST=1 ** Save this variables*** #avgdata_mat = np.hstack([X_Sensor_STATE,X_Weather_STATE,X_Time_STATE]) #avgdata_names = X_Sensor_NAMES+X_Weather_NAMES+X_Time_NAMES avgdata_exemplar = dict(sf_exemplars_dict.items()+si_exemplars_dict.items()) avgdata_zvar = X_zero_var_list avgdata_dict = dict() avgdata_dict.update({'build_feature_matrix_out': build_feature_matrix_out}) avgdata_dict.update({'avgdata_state_mat': X_Sensor_STATE}) avgdata_dict.update({'avgdata_weather_mat': X_Weather_STATE}) avgdata_dict.update({'avgdata_time_mat': X_Time_STATE}) avgdata_dict.update({'avg_time_slot': X_Time}) avgdata_dict.update({'avgdata_exemplar': avgdata_exemplar}) avgdata_dict.update({'avgdata_zvar': avgdata_zvar}) avgdata_dict.update({'sensor_names': X_Sensor_NAMES}) avgdata_dict.update({'weather_names': X_Weather_NAMES}) avgdata_dict.update({'time_names': X_Time_NAMES}) dsout.update({'avgdata_dict': avgdata_dict}) if proc_diff: log.info('-' * 40) log.info('processing diff.feature..') log.info('-' * 40) #################################### # Irregular Event Extraction #################################### # Interpolatoin with outlier removal, Here we exclude weather data from irregualr event analysis # since weather data noramlly show slow changes in time.so we dont expect in any meaningful diffs values measurement_point_set,num_type_set = interpolation_measurement(data_dict, sensor_list, err_rate=1, sgm_bnd=20) # Irregualr matrix Xdiff_Mat,\ Xdiff_Time,\ Xdiff_Names,\ Xdiff_zero_var_list,\ Xdiff_zero_var_val,\ Xdiff_int_type_list,\ Xdiff_int_type_idx,\ Xdiff_float_type_list,\ Xdiff_float_type_idx =\ build_diff_matrix(measurement_point_set, time_slots, num_type_set, sensor_list, PARALLEL=PARALLEL) build_diff_matrix_out = \ {'Xdiff_Mat':Xdiff_Mat, 'Xdiff_Time':Xdiff_Time, 'Xdiff_Names':Xdiff_Names, 'Xdiff_zero_var_list':Xdiff_zero_var_list, 'Xdiff_zero_var_val':Xdiff_zero_var_val, 'Xdiff_int_type_list':Xdiff_int_type_list, 'Xdiff_int_type_idx':Xdiff_int_type_idx, 'Xdiff_float_type_list':Xdiff_float_type_list, 'Xdiff_float_type_idx':Xdiff_float_type_idx} build_diff_matrix_out = obj(build_diff_matrix_out) if Xdiff_Mat.shape[0] > 0: #============================================================================== # Restructure diff_marix's and weather matix for the same common time slot #============================================================================== time_slots_array = np.sort(np.array(list(set(Xdiff_Time) & set(X_Time)))) # Extract subset of X_Weather_STATE removed_idx_list = list() for ridx, slot in enumerate(X_Time): slot_idx = np.where(time_slots_array==slot)[0] # slot not in common time slots if len(slot_idx) == 0: removed_idx_list.append(ridx) XDIFF_Weather_STATE = np.delete(X_Weather_STATE, removed_idx_list,axis=0) # Extract subset of Xdiff_Mat removed_idx_list = list() for ridx,slot in enumerate(Xdiff_Time): slot_idx = np.where(time_slots_array == slot)[0] # slot not in common time slots if len(slot_idx) == 0: removed_idx_list.append(ridx) Xdiff_Mat = np.delete(Xdiff_Mat, removed_idx_list, axis=0) # Update Xdiff_Time Xdiff_Time = time_slots_array XDIFF_Weather_STATE = np.array(XDIFF_Weather_STATE) # From below all index are reference to X_Feature xdiff_sf_idx = Xdiff_float_type_idx xdiff_sf_name = Xdiff_float_type_list xdiff_si_idx = Xdiff_int_type_idx xdiff_si_name = Xdiff_int_type_list # Float Type Measurement Clustering X_Diff_sfe, sf_diff_exemplars_dict, exemplars_, labels_ = \ cluster_measurement_points(Xdiff_Mat[:, xdiff_sf_idx], xdiff_sf_name, corr_bnd=[0.1, 0.9]) xdiff_sfe_idx = list(np.array(xdiff_sf_idx)[exemplars_]) # InT Type Measurement Clustering X_Diff_sie, si_diff_exemplars_dict, exemplars_, labels_ = \ cluster_measurement_points(Xdiff_Mat[:, xdiff_si_idx], xdiff_si_name, corr_bnd=[0.1, 0.9]) xdiff_sie_idx = list(np.array(xdiff_si_idx)[exemplars_]) # sensor -float type xdiff_sfe_state, xdiff_sfe_corr_val =\ x_input_to_states(X_Diff_sfe, CORR_VAL_OUT=1, PARALLEL=PARALLEL) # sensor -integer type xdiff_sie_state = X_Diff_sie empty_states = np.array([[] for i in range(len(Xdiff_Time))]) if len(xdiff_sfe_state) == 0: xdiff_sfe_state = empty_states if len(xdiff_sie_state) == 0: xdiff_sie_state = empty_states if len(wf_state) == 0: wf_state = empty_states if len(wi_state) == 0: wi_state = empty_states # Exemplar sensor only XDIFF_Sensor_STATE = np.append(xdiff_sfe_state,xdiff_sie_state, axis=1) XDIFF_Sensor_STATE = XDIFF_Sensor_STATE.astype(int) XDIFF_Sensor_NAMES = list(np.array(Xdiff_Names)[xdiff_sfe_idx])+list(np.array(Xdiff_Names)[xdiff_sie_idx]) # months of a year,days of a week, and hours of a day # (Monday, Tuesday,Wendsday,Thursday,Saturday,Sunday) =(0,1,2,3,4,5,6) XDIFF_Time_STATE_temp = build_time_states(Xdiff_Time) XDIFF_Time_NAMES_temp = ['MTH', 'WD', 'HR'] XDIFF_Time_STATE = list() XDIFF_Time_NAMES = list() for xt_col, xt_name in zip(XDIFF_Time_STATE_temp.T, XDIFF_Time_NAMES_temp): if len(set(xt_col)) > 1: XDIFF_Time_STATE.append(xt_col) XDIFF_Time_NAMES.append(xt_name) XDIFF_Time_STATE = np.array(XDIFF_Time_STATE).T ################################################# # FORMATTED DATA FOR IRREGUALR EVENT ################################################# #** Save this variables*** #diffdata_mat = np.hstack([XDIFF_Sensor_STATE,X_Weather_STATE,XDIFF_Time_STATE]) #diffdata_names = XDIFF_Sensor_NAMES+X_Weather_NAMES+XDIFF_Time_NAMES diffdata_exemplar = dict(sf_diff_exemplars_dict.items() + si_diff_exemplars_dict.items()) diffdata_zvar = Xdiff_zero_var_list diffdata_dict = dict() diffdata_dict.update({'build_diff_matrix_out': build_diff_matrix_out}) diffdata_dict.update({'diffdata_state_mat': XDIFF_Sensor_STATE}) diffdata_dict.update({'diffdata_weather_mat': XDIFF_Weather_STATE}) diffdata_dict.update({'diffdata_time_mat': XDIFF_Time_STATE}) diffdata_dict.update({'diff_time_slot': Xdiff_Time}) diffdata_dict.update({'diffdata_exemplar': diffdata_exemplar}) diffdata_dict.update({'diffdata_zvar': diffdata_zvar}) diffdata_dict.update({'sensor_names': XDIFF_Sensor_NAMES}) diffdata_dict.update({'weather_names': X_Weather_NAMES}) diffdata_dict.update({'time_names': X_Time_NAMES}) dsout.update({'diffdata_dict': diffdata_dict}) dsout.update({'bldg_key': remove_dot(bldg_key)}) return dsout
def build_diff_matrix(measurement_point_set, time_slots, num_type_set, irr_data_name, conf_lev=0.5, PARALLEL=False): #time_slots_utc = dtime_to_unix(time_slots) Xdiff = list() input_names = list() INT_type_list = list() FLOAT_type_list = list() INT_type_idx = list() FLOAT_type_idx = list() zero_var_list = list() # whose variance is zero, hence carry no information, zero_var_val = list() num_of_samples = len(time_slots) #TIMELET_INV_seconds = (time_slots[1]-time_slots[0]).seconds log.info('=' * 40) if not PARALLEL: for k, (set_val, set_name) in enumerate(zip(measurement_point_set, irr_data_name)): log.info(str(irr_data_name[k])) try: num_type = num_type_set[k] diff_mean = get_diff(set_val, time_slots, num_type, conf_lev) if num_type == FLOAT_TYPE: #norm_diff_mean,output_status=normalize_data(diff_mean[:,0]) norm_diff_mean, output_status = normalize_data(diff_mean) elif num_type == INT_TYPE: #num_discrete_vals=len(set(list(diff_mean[:,0]))) num_discrete_vals = len(set(list(diff_mean))) log.info('num_discrete_vals : ' + str(num_discrete_vals)) if num_discrete_vals > 1: output_status = 0 norm_diff_mean = diff_mean else: output_status = -1 #norm_diff_mean = list(set(diff_mean[:,0])) norm_diff_mean = list(set(diff_mean)) else: pass if len(np.nonzero(norm_diff_mean == np.inf)[0])>num_of_samples/5: raise except Exception as e: log.error(traceback.print_exc()) log.error('Error in processing data feature, excluded from analysis ' + str(e)) output_status = -1 norm_diff_mean = None if output_status == -1: #zero_var_flag=1 zero_var_list.append(set_name) zero_var_val.append(norm_diff_mean) log.warn('too small variance for float type or a single value for int type, added to zero var list') else: input_names.append(set_name) Xdiff.append(norm_diff_mean) if num_type == FLOAT_TYPE: FLOAT_type_list.append(set_name) FLOAT_type_idx.append(len(Xdiff)-1) elif num_type == INT_TYPE: INT_type_list.append(set_name) INT_type_idx.append(len(Xdiff)-1) log.info('-' * 20) log.info('-' * 40) # PARALLEL ENABLED else: log.info('Build diff matrix: Parallel enabled...') # Construct param list for workers param_list = list() for k, (set_val, set_name) in enumerate(zip(measurement_point_set, irr_data_name)): param_list.append((k, time_slots, conf_lev, set_val, set_name, num_type_set[k])) p = mp.Pool(CPU_CORE_NUM) ret_dict = dict(p.map(build_diff, param_list)) p.close() p.join() for k in sorted(ret_dict.keys()): """ v = ret_dict[k] output_status = v[0] norm_diff_mean = v[1] """ output_status, norm_diff_mean = ret_dict[k] set_name = irr_data_name[k] num_type = num_type_set[k] if output_status == -1: zero_var_list.append(set_name) #zero_var_flag=1 zero_var_val.append(norm_diff_mean) log.warn("too small variance for float type or a single value for int type, added to zero var list") else: input_names.append(set_name) try: Xdiff.append(norm_diff_mean) except Exception as e: log.error(traceback.print_exc()) log.error(str(e)) if num_type == FLOAT_TYPE: FLOAT_type_list.append(set_name) FLOAT_type_idx.append(len(Xdiff)-1) elif num_type == INT_TYPE: INT_type_list.append(set_name) INT_type_idx.append(len(Xdiff)-1) log.info('-' * 20) Xdiff = np.array(Xdiff).T deleted_timeslot_idx = list() log.info('-' * 20) log.info('removing time slots having no sample...') inf_idx_set = list() for col_vec in Xdiff.T: inf_idx = np.nonzero(col_vec == np.infty)[0] inf_idx_set=np.r_[inf_idx_set, inf_idx] inf_col_idx = list(set(list(inf_idx_set))) deleted_timeslot_idx = np.array([int(x) for x in inf_col_idx]).astype(int) log.info('time slots ' + str(deleted_timeslot_idx) + ' removed...') log.info('-' * 20) Xdiff = np.delete(Xdiff, deleted_timeslot_idx, axis=0) new_time_slot = np.delete(time_slots, deleted_timeslot_idx) # Checking whether it has any ill entry value verify_data_mat(Xdiff) log.info('*-' * 20) log.info("* deleted_timeslot_idx : " + str(deleted_timeslot_idx)) log.info('*-' * 20) return Xdiff,\ new_time_slot,\ input_names,\ zero_var_list,\ zero_var_val, \ INT_type_list,\ INT_type_idx,\ FLOAT_type_list,\ FLOAT_type_idx
def state_retrieval(obs, max_num_cluster=6, off_set=0, est_method='kmean', PARALLEL = False): log.info('-' * 40) log.info('Retrieving discrete states from data using ' + est_method + ' model...') log.info('-' * 40) log.info('try '+ str(max_num_cluster) + ' clusters..... ') score = np.zeros(max_num_cluster) model_set = list() if not PARALLEL: for num_cluster in range(max_num_cluster): log.info('Try ' + str(num_cluster+1) + ' clusters ') log.info('-----------------------------------') if est_method == 'kmean': kmean = KMeans(n_clusters=num_cluster+1).fit(obs) model_set.append(kmean) #score[num_cluster]=-1*np.log(-1*np.sum(kmean.score(obs))) #score[num_cluster]=kmean.score(obs) #score[num_cluster]=kmean.score(obs)-.5*(num_cluster+1)*1*log10(len(obs)) #log_ll_val=compute_log_ll(kmean.labels_,obs) score[num_cluster] = compute_log_ll(kmean.labels_, obs) elif est_method == 'gmm': gmm = mixture.GMM(n_components=num_cluster+1).fit(obs) model_set.append(gmm) score[num_cluster] = np.sum(gmm.score(obs)) else: log.error('not supported est_method') raise NameError('not supported est_method') else: log.info('Parallel enabled...') model_set = [0] * max_num_cluster score = [0] * max_num_cluster params = [(obs, i+1, est_method) for i in range(max_num_cluster)] p = Pool(max_num_cluster) models = p.map(pp_cluster_state_retrieval, params) p.close() p.join() model_dict = dict(models) for k, v in model_dict.iteritems(): model_set[k] = v[0] score[k] = v[1] score_err_sum = np.zeros(max_num_cluster) log.info('Finding knee points of log likelihood...') for i in range(max_num_cluster): a_0 = score[:(i)] if len(a_0) > 1: slope, intercept, r_value, p_value, std_err = stats.linregress(range(len(a_0)),a_0) sqr_sum_err0 = sum(((slope*np.arange(len(a_0)) + intercept)-a_0)**2) else: sqr_sum_err0=0 a_1 = score[(i):] if len(a_1) > 1: slope, intercept, r_value, p_value, std_err = stats.linregress(range(len(a_1)),a_1) sqr_sum_err1 = sum(((slope*np.arange(len(a_1)) + intercept)-a_1)**2) else: sqr_sum_err1 = 0 score_err_sum[i] = sqr_sum_err0 + sqr_sum_err1 # Optimum number of clusters. min_idx = np.argmin(score_err_sum) opt_num_cluster = min_idx+1 log.info('opt_num_cluster: ' + str(opt_num_cluster)) if est_method == 'kmean': label = model_set[min_idx].labels_ elif est_method == 'gmm': label = model_set[min_idx].predict(obs) else: raise NameError('not supported est_method') return label, opt_num_cluster, model_set[min_idx], score, score_err_sum
def bn_probability_analysis(bldg_obj, sig_tag='avg'): p_name_set = [analysis.sensor_tag for analysis in bldg_obj.analysis[sig_tag]] if sig_tag == "avg": event = bldg_obj.sigtags['avg'].weather_dict['Event'] cond = bldg_obj.sigtags['avg'].weather_dict['Cond'] elif sig_tag == "diff": event = bldg_obj.sigtags['diff'].weather_dict['Event'] cond = bldg_obj.sigtags['diff'].weather_dict['Cond'] bn_out_set = list() for p_name in p_name_set: try: # bn analysis - Power-Sensor s_cause_label, s_labels, s_hc, s_cp_mat, s_bndata_mat = \ _bn_anaylsis(bldg_obj, p_name, attr='sensor', sig_tag=sig_tag, num_picks_bn=5) # bn analysis -Power-Time t_cause_label, t_labels, t_hc, t_cp_mat, t_bndata_mat = \ _bn_anaylsis(bldg_obj, p_name, attr='time', sig_tag=sig_tag, num_picks_bn=10) # bn analysis -Power-Weather w_cause_label, w_labels, w_hc, w_cp_mat, w_bndata_mat = \ _bn_anaylsis(bldg_obj, p_name, attr='weather', sig_tag=sig_tag,num_picks_bn=10) # bn analysis -Power-Sensor+Time+Weather all_cause_label, all_labels, all_hc, all_cp_mat, all_bndata_mat=\ _bn_anaylsis_all(bldg_obj, p_name, sig_tag=sig_tag, num_picks_bn=20) # prob analysis -Power-Sensor+Time+Weather cause_label = all_cause_label col_labels = all_labels effect_label = p_name bndata_mat = all_bndata_mat low_peak_state, low_peak_prob, high_peak_state, high_peak_prob = \ _peak_analysis(cause_label, effect_label, col_labels, bndata_mat) x_set = low_peak_state all_cause_symbol_xlabel = _get_tick_symbol(x_set, all_cause_label, event, cond) all_cause_symbol_xtick = range(len(low_peak_state)) # BN-PROB STORE bn_out = \ {'p_name': p_name, 's_cause_label': s_cause_label, 's_labels': s_labels, 's_hc': s_hc, 's_cp_mat': s_cp_mat, 's_bndata_mat': s_bndata_mat, 't_cause_label': t_cause_label, 't_labels': t_labels, 't_hc': t_hc, 't_cp_mat': t_cp_mat, 't_bndata_mat': t_bndata_mat, 'w_cause_label': w_cause_label, 'w_labels': w_labels, 'w_hc': w_hc, 'w_cp_mat': w_cp_mat, 'w_bndata_mat': w_bndata_mat, 'all_cause_label': all_cause_label, 'all_labels': all_labels, 'all_hc': all_hc, 'all_cp_mat': all_cp_mat, 'all_bndata_mat': all_bndata_mat, 'low_peak_state': low_peak_state, 'low_peak_prob': low_peak_prob, 'high_peak_state': high_peak_state, 'high_peak_prob': high_peak_prob, 'all_cause_symbol_xlabel': all_cause_symbol_xlabel, 'all_cause_symbol_xtick': all_cause_symbol_xtick} bn_out_set.append(bn_out) except Exception as e: log.error('*** Error in processing bn_prob for ' + p_name + '! ****') log.error(traceback.print_exc()) log.error(str(e)) pass return bn_out_set