def getCountriesOverview(self): countries = {} for country in MediationConfig.getCountryList(): allStatuses = self.getAll(country) ok = 0 warning = 0 outage = 0 expired = 0 disabled = 0 for lobName, lob in MediationConfig.getLobs(country).items(): for flow in lob["flows"].values(): gName = flow["gName"] if flow["options"]["enabled"] is False: disabled += 1 continue flowStatus = allStatuses.get( gName, {"status": status.NA})["status"] if flowStatus == status.NA: expired += 1 elif flowStatus == status.OK: ok += 1 elif flowStatus == status.WARNING: warning += 1 elif flowStatus == status.OUTAGE: outage += 1 countries[country] = { status.OK: ok, status.WARNING: warning, status.OUTAGE: outage, status.NA: expired, status.DISABLED: disabled } return countries
def putCountriesConfig(): body = request.get_json() MediationConfig.getCollection().update_one(MEDIATION_DOCUMENT, {"$set": { "countries": body }}) return jsonify(MediationConfig.getCountries())
def deleteLob(country, lobName): lob = MediationConfig.getLobWithCountry(country, lobName) if lob is not None: MediationConfig.deleteLob(lob) else: raise StatusException("lob does not exists", 400) return getCountry(country)
def enableLob(country, lobName): enable = request.get_json()["enable"] optionsPath = "lobs." + country + "." + lobName + ".options.enabled" MediationConfig.getCollection().update_one(MEDIATION_DOCUMENT, {"$set": { optionsPath: enable }}) return jsonify( MediationConfig.getLobWithCountry(country, lobName)["options"])
def lobOptionsPUT(country, lobName): body = request.get_json() optionsPath = "lobs." + country + "." + lobName + ".options" MediationConfig.getCollection().update_one(MEDIATION_DOCUMENT, {"$set": { optionsPath: body }}) return jsonify( MediationConfig.getLobWithCountry(country, lobName)["options"])
def enableCountry(country): body = request.get_json() enable = body["enable"] MediationConfig.getCollection().update_one( MEDIATION_DOCUMENT, {"$set": { "countries." + country + ".enabled": enable }}) return getEnabledCountry(country)
def addLob(): """put under /lobs""" addLobRequest = request.get_json() country = addLobRequest["country"] lobName = addLobRequest["lobName"] if MediationConfig.getLobWithCountry(country, lobName) is not None: raise StatusException("Lob already exists", 400) MediationConfig.addLob(country, lobName) return getCountry(country)
def addFlow(country, lobName): lob = MediationConfig.getLobWithCountry(country, lobName) addFlowRequest = request.get_json() name = addFlowRequest["name"] type = addFlowRequest["type"] if name in lob["flows"]: raise StatusException("Flow already exists", 400) flow = {"country": country, "lobName": lobName, "type": type, "name": name} MediationConfig.addFlow(flow) return getLob(country, lobName)
def putFlowOptions(country, lobName, flowName): body = request.get_json() flow = MediationConfig.getLobWithCountry(country, lobName)["flows"][flowName] res = MediationConfig.getCollection().update_one( MEDIATION_DOCUMENT, {"$set": { "lobs." + flow["dataPath"]: body }}) return jsonify( MediationConfig.getLobWithCountry( country, lobName)["flows"][flowName]["options"])
def enableFlow(country, lobName, flowName): body = request.get_json() enable = body["enable"] flow = MediationConfig.getLobWithCountry(country, lobName)["flows"][flowName] MediationConfig.getCollection().update_one( MEDIATION_DOCUMENT, {"$set": { "lobs." + flow["dataPath"] + ".enabled": enable }}) return jsonify( MediationConfig.getLobWithCountry( country, lobName)["flows"][flowName]["options"])
def _executeInternal(self): flowQueue = Queue() for country in MediationConfig.getCountryList(): self._enqueFlowsToAnalyze(flowQueue, country) if not flowQueue.empty(): start = time.time() flowsToAnalyzeCnt = flowQueue.unfinished_tasks logging.info("Going to analyze " + str(flowsToAnalyzeCnt)+" flows") workers = [Worker(flowQueue, self.statusManager) for i in range(0, MediationConfig.threadsCount())] for worker in workers: worker.start() for worker in workers: worker.join() logging.info("Finished analyzing " + str(flowsToAnalyzeCnt) + " flows. Time: " + str(int(time.time() - start)) + " seconds.")
def _executeInternal(self): start = time.time() cursor = mongo.traffic().find({ "$and": [{ "_id": { "$gte": self.fromDate } }, { "_id": { "$lt": self.toDate } }] }) allFlows = dict((countryName, {}) for countryName in MediationConfig.getCountries()) currentConfig = dict( (countryName, MediationConfig.getLobs(countryName)) for countryName in MediationConfig.getCountries()) newFlows = {} newLobs = dict((countryName, {}) for countryName in MediationConfig.getCountries()) for doc in cursor: for countryName, country in doc["data"].items(): for lobName, lob in country.items(): if lobName == "FOX": continue if currentConfig[countryName].get(lobName, None) == None: newLobs[countryName][lobName] = True for flowName, flow in lob.get("inputs", {}).items(): if flowName == "updatesCnt" or flowName == "sum": continue if flowName not in currentConfig[countryName].get( lobName, {}).get("inputs", {}): newFlows[createGlobalName(countryName, lobName, flowName, "inputs")] = True for flowName, flow in lob.get("forwards", {}).items(): if flowName == "updatesCnt" or flowName == "sum": continue if flowName not in currentConfig[countryName].get( lobName, {}).get("forwards", {}): newFlows[createGlobalName(countryName, lobName, flowName, "forwards")] = True # print(flowName) insertedLobs = 0 for countryName, lobs in newLobs.items(): for lobName in lobs.keys(): insertedLobs += 1 MediationConfig.addLob(countryName, lobName) for globalFlowName in newFlows.keys(): flow = globalNameToFlow(globalFlowName) MediationConfig.addFlow(flow) if insertedLobs > 0 or len(newFlows) > 0: logging.info("inserted " + str(insertedLobs) + " lobs and " + str(len(newFlows)) + " flows in " + str(int(time.time() - start)) + " seconds")
def flowGET(country, lobName, flowName): flow = MediationConfig.getLobWithCountry(country, lobName)["flows"][flowName] if shouldIncludeStatus(): status = FlowStatusManager().getLobDetailWithCountry(country, lobName)[flowName] flow["status"] = status return jsonify(flow)
def getLobDetail(self, lobName): lob = MediationConfig.getLob(lobName) allStatuses = self.getAll(lob["country"]) lobFlowStatuses = {} for flowName, flow in lob["flows"].items(): lobFlowStatuses[flowName] = allStatuses[flow["gName"]] return lobFlowStatuses
def _enqueFlowsToAnalyze(self, flowQueue, country): self.lastExecutions = self.statusManager.getAll(country) countryLobs = MediationConfig.getLobs(country, enabledOnly=True) for lobName, lob in countryLobs.items(): for flow in lob["flows"].values(): if self.shouldSchedule(flow): job = {"flow": flow, "lastExecution": self.lastExecutions[flow["gName"]]} flowQueue.put(job)
def _getFlowsToAnalyze(self, country): flowsToAnalyze = [] lobsConfig = MediationConfig.getLobs(country, enabledOnly=True) for lobName, lob in lobsConfig.items(): for flow in lob["flows"].values(): if self.shouldSchedule(flow): flowsToAnalyze.append(flow) return flowsToAnalyze
def getLob(country, lobName): res = MediationConfig.getLobWithCountry(country, lobName) if shouldIncludeStatus(): status = FlowStatusManager().getLobDetailWithCountry(country, lobName) for k, v in status.items(): type = res["flows"][k]["type"] res[type][k]["status"] = v del res["flows"] return jsonify(res)
def __init__(self, flow): self.options = flow["options"] self.independentDays = self.options.get("independentDays", []) self.country = MediationConfig.getCountryByName(flow["country"]) self.HOLIDAYS = [] for holiday in self.country["holidays"]: day, month = holiday.split(".") self.HOLIDAYS.append((int(day), int(month))) pass
def getCountry(country): res = MediationConfig.getLobs(country) for lob in res.values(): del lob["flows"] del lob["inputs"] del lob["forwards"] if shouldIncludeStatus(): status = FlowStatusManager().getLobsOverview(country) addStatus(res, status) return jsonify(res)
def getAll(self, country): lobs = MediationConfig.getLobs(country) res = mongo.statuses().find_one({"_id": "lobs"}, {"_id": 0}) if res is None: res = {} statuses = {} for lobName, lob in lobs.items(): for flowName, flow in lob["flows"].items(): gName = flow["gName"] if flow["options"]["enabled"] is False: statuses[gName] = {"status": status.DISABLED} elif gName in res: statuses[gName] = self._setStatusMetadata(res[gName], flow) else: statuses[gName] = {"status": status.NA} return statuses
def __init__(self, date, flows, granularity=0): self.flows = flows self.flow = flows[0] self.date = date self.granularity = granularity self.dates = SimilarPastDaysFinder(self.flow).findSimilarPastDays(date) country = MediationConfig.getCountryByName(self.flow["country"]) self.options = self.flow["options"] self.adjustment = None lazyDayDifference = self.flow["options"].get("lazyDayDifference", None) if lazyDayDifference != None: for lazyDayStr in country["lazyDays"]: lazyDay = util.stringToDate(lazyDayStr) if lazyDay.date() == date.date(): self.adjustment = lazyDayDifference break
def getCountriesConfig(): return jsonify(MediationConfig.getCountries())
def getEnabledCountry(country): return jsonify( {"enabled": MediationConfig.getCountryByName(country)["enabled"]})
def dataQueryV2(): """ Endpoint for getting traffic data. POST body: { "from":"01.02.2017", "to":"15.02.2017", "country":"CZ", "lobName":"ACI", "flowName":["GSM"], "forwards":[], "granularity":0 } Response: { "data": [ { "GSM": 188385532, "_id": "2017-02-01T00:00:00+01:00", "dayAverage": 1162595297.6666667, "dayDifference": 1.023, "expected": 161627916, "status": "OK", "tickDifference": 1.166 }, ... ], "metadata": { "flowName": "GSM", "granularity": 480, "metrics": { "GSM": { "type": "traffic" }, "dayAverage": { "type": "traffic" }, "dayDifference": { "type": "difference" }, "expected": { "type": "traffic" }, "status": { "type": "other" }, "tickDifference": { "type": "difference" } } } """ searchParam = request.get_json() fromDate = util.stringToDate(searchParam["from"]) toDate = util.stringToDate(searchParam["to"]) country = searchParam["country"] lobName = searchParam["lobName"] lobConfig = MediationConfig.getLobWithCountry(country, lobName) flows = [] granularity = searchParam.get("granularity", 0) flows.append(lobConfig["flows"][searchParam["flowName"]]) response = {} # Query the traffic data and add to metric list mongoQuery = data_query.DateRangeGroupQuery(fromDate, toDate, flows, granularity=granularity) data = mongoQuery.execute() metrics = {} metricsList = [] flowName = mongoQuery.metrics[0] metricsList.append(flowName) metadata = mongoQuery.metadata if len(flows) == 1: # Run outage detection analysis metric = metricsList[0] flowLevelQuery = data_query.FlowLevelDateRangeQuery( fromDate, toDate, flows, metadata["granularity"], data) flowLevelData = flowLevelQuery.execute() data = util.merge2DateLists(flowLevelData, None, data, None) metricsList.extend(flowLevelQuery.metrics) outageQuery = data_query.OutageDateRangeQuery(fromDate, toDate, flows[0], metadata["granularity"]) outageQuery.setPrecomputedData(data, metric) outageList = outageQuery.execute() data = util.merge2DateLists(outageList, [outageQuery.metric], data, None) metricsList.append(outageQuery.metric) # Create metadata infor for metric in metricsList: if metric == flowName or metric == "dayAverage" or metric == "expected": type = "traffic" elif "Difference" in metric: type = "difference" else: type = "other" metrics[metric] = {"type": type} response["data"] = data response["metadata"] = { **{ "metrics": metrics }, **metadata, "flowName": flowName } return jsonify(response)
windowData = data[start:i + 1] weights = [1 + x * x for x in range(0, len(windowData))] s = 0 for x, y in zip(windowData, weights): s += x * y average = s / sum(weights) output.append(average) return output fromDate = util.stringToTime("02.03.2017 00:00:00") toDate = util.stringToTime("03.03.2017 00:00:00") granularity = 15 q = DateRangeGroupQuery( fromDate, toDate, [MediationConfig.getLobWithCountry("CZ", "GSM")["flows"]["MSSBRN1A"]], granularity) # q = DateRangeGroupQuery(fromDate, toDate, [MediationConfig.getLobWithCountry("CZ","ACI")["flows"]["GSM"]], granularity) lob1Data = util.dateDataListToList(q.execute(), "MSSBRN1A") lob1Data.append(0) lob1Data.append(0) lob1Data.append(0) lob1Data.append(0) dates = list( map( lambda x: util.dateToTimeString(x).split("+")[0].split("T")[1].split( ":")[0], util.dateDataListToList(q.execute(), "_id"))) dataTicks = [] dataTickPos = [] for i in range(0, len(dates)): if i % 4 == 0:
dataTicks = [] dataTickPos = [] for i in range(0, len(dates)): if True: dataTicks.append(dates[i]) dataTickPos.append(i) return data1, expected, dataTicks, dataTickPos fromDate = util.stringToDate("30.01.2017") toDate = util.stringToDate("12.02.2017") # lobName = "ICG" # flowName = "CENTREX01" lobName = "GSM" flowName = "MSSBRN1B" lob = MediationConfig.getLobWithCountry("CZ", lobName) flow = lob["flows"][flowName] flows = [flow] granularity = 240 data = [] expected = [] ticks = [] pos = [] d, e, t, p = getData(flows, util.stringToDate("28.09.2016"), util.stringToDate("29.09.2016")) data += d expected += e ticks += t posLen = len(pos) pos += [i + posLen for i in p]
def getFlowOptions(country, lobName, flowName): lobConfig = MediationConfig.getLobWithCountry(country, lobName) flowConfig = lobConfig["flows"][flowName] return jsonify(flowConfig)
def deleteFlow(country, lobName, flowName): lob = MediationConfig.getLobWithCountry(country, lobName) if flowName in lob["flows"]: MediationConfig.deleteFlow(lob["flows"][flowName]) return getLob(country, lobName)
0.796042342418818, 0.9346540748162848, 0.7939754809772508, 0.962470501562014, 0.9929573860362824, 1]}, {"lobName": 'EWG', "data": [0.9761478362504887, 0.9583614156352167, 1.2336652720530112, 1.1580357149388305, 1.090568389327002, 0.9600084918076629, 0.9568424857376663, 0.8256895883325289, 0.5914861789815206, 1.0121511182076122, 0.8653533492971078, 0.5704172842604073, 0.538653445993039, 0.5164638232491638, 0.4858079169661382, 0.41779533433377064, 0.7993573176963311, 0.9092267083181577, 1.007730199059085, 1.0702191864202208, 0.9082408024540793, 1]}, {"lobName": 'TPP', "data": [0.9994509893629189, 0.9329863891112891, 0.8681779342746304, 0.8697074230973179, 0.8752423267949921, 0.8735428814090898, 0.872957061838658, 0.8761341580186848, 0.8998970751542529, 0.8653601909437505, 0.8668891761965745, 0.7537979083127982, 0.9302389942504077, 0.9967766261759137, 0.9968273519280757, 0.9897328514978554, 0.8680083180866294, 0.9732343917814772, 0.9873704480552615, 0.9832499869540516, 0.9831210955086642, 1]}] CZ_LOBS = MediationConfig.getCountryByName("CZ") # CZ_LOBS = ["SMS", "GSM", "MMS"] def getnormalizedFlowLevel(lobName, granularity=0): fromDate = util.jsStringToDate("2016-12-15T00:00:00.000Z").replace(hour=0, minute=0, second=0) toDate = util.jsStringToDate("2017-01-05T00:00:00.000Z").replace(hour=0, minute=0, second=0) response = {} mongoQuery = data_query.DateRangeGroupQuery(fromDate, toDate, [lobName], granularity) data = mongoQuery.execute() metrics = {} metricsList = mongoQuery.metrics metadata = mongoQuery.metadata data = data_query.medianDateRange(fromDate, toDate, lobName, granularity, data) return util.dateDataListToList(data, "normalizedFlowLevel")
def resetFlowStatus(country, lobName, flowName): flow = MediationConfig.getFlow(country, lobName, flowName) FlowStatusManager().resetStatus(flow) return flowGET(country, lobName, flowName)