def getCountriesOverview(self):
countries = {}
for country in MediationConfig.getCountryList():
allStatuses = self.getAll(country)
ok = 0
warning = 0
outage = 0
expired = 0
disabled = 0
for lobName, lob in MediationConfig.getLobs(country).items():
for flow in lob["flows"].values():
gName = flow["gName"]
if flow["options"]["enabled"] is False:
disabled += 1
continue
flowStatus = allStatuses.get(
gName, {"status": status.NA})["status"]
if flowStatus == status.NA:
expired += 1
elif flowStatus == status.OK:
ok += 1
elif flowStatus == status.WARNING:
warning += 1
elif flowStatus == status.OUTAGE:
outage += 1
countries[country] = {
status.OK: ok,
status.WARNING: warning,
status.OUTAGE: outage,
status.NA: expired,
status.DISABLED: disabled
}
return countries
def putCountriesConfig():
body = request.get_json()
MediationConfig.getCollection().update_one(MEDIATION_DOCUMENT,
{"$set": {
"countries": body
}})
return jsonify(MediationConfig.getCountries())
def deleteLob(country, lobName):
lob = MediationConfig.getLobWithCountry(country, lobName)
if lob is not None:
MediationConfig.deleteLob(lob)
else:
raise StatusException("lob does not exists", 400)
return getCountry(country)
def enableLob(country, lobName):
enable = request.get_json()["enable"]
optionsPath = "lobs." + country + "." + lobName + ".options.enabled"
MediationConfig.getCollection().update_one(MEDIATION_DOCUMENT,
{"$set": {
optionsPath: enable
}})
return jsonify(
MediationConfig.getLobWithCountry(country, lobName)["options"])
def lobOptionsPUT(country, lobName):
body = request.get_json()
optionsPath = "lobs." + country + "." + lobName + ".options"
MediationConfig.getCollection().update_one(MEDIATION_DOCUMENT,
{"$set": {
optionsPath: body
}})
return jsonify(
MediationConfig.getLobWithCountry(country, lobName)["options"])
def enableCountry(country):
body = request.get_json()
enable = body["enable"]
MediationConfig.getCollection().update_one(
MEDIATION_DOCUMENT,
{"$set": {
"countries." + country + ".enabled": enable
}})
return getEnabledCountry(country)
def addLob():
"""put under /lobs"""
addLobRequest = request.get_json()
country = addLobRequest["country"]
lobName = addLobRequest["lobName"]
if MediationConfig.getLobWithCountry(country, lobName) is not None:
raise StatusException("Lob already exists", 400)
MediationConfig.addLob(country, lobName)
return getCountry(country)
def addFlow(country, lobName):
lob = MediationConfig.getLobWithCountry(country, lobName)
addFlowRequest = request.get_json()
name = addFlowRequest["name"]
type = addFlowRequest["type"]
if name in lob["flows"]:
raise StatusException("Flow already exists", 400)
flow = {"country": country, "lobName": lobName, "type": type, "name": name}
MediationConfig.addFlow(flow)
return getLob(country, lobName)
def putFlowOptions(country, lobName, flowName):
body = request.get_json()
flow = MediationConfig.getLobWithCountry(country,
lobName)["flows"][flowName]
res = MediationConfig.getCollection().update_one(
MEDIATION_DOCUMENT, {"$set": {
"lobs." + flow["dataPath"]: body
}})
return jsonify(
MediationConfig.getLobWithCountry(
country, lobName)["flows"][flowName]["options"])
def enableFlow(country, lobName, flowName):
body = request.get_json()
enable = body["enable"]
flow = MediationConfig.getLobWithCountry(country,
lobName)["flows"][flowName]
MediationConfig.getCollection().update_one(
MEDIATION_DOCUMENT,
{"$set": {
"lobs." + flow["dataPath"] + ".enabled": enable
}})
return jsonify(
MediationConfig.getLobWithCountry(
country, lobName)["flows"][flowName]["options"])
def _executeInternal(self):
flowQueue = Queue()
for country in MediationConfig.getCountryList():
self._enqueFlowsToAnalyze(flowQueue, country)
if not flowQueue.empty():
start = time.time()
flowsToAnalyzeCnt = flowQueue.unfinished_tasks
logging.info("Going to analyze " + str(flowsToAnalyzeCnt)+" flows")
workers = [Worker(flowQueue, self.statusManager) for i in range(0, MediationConfig.threadsCount())]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
logging.info("Finished analyzing " + str(flowsToAnalyzeCnt) +
" flows. Time: " + str(int(time.time() - start)) + " seconds.")
def _executeInternal(self):
start = time.time()
cursor = mongo.traffic().find({
"$and": [{
"_id": {
"$gte": self.fromDate
}
}, {
"_id": {
"$lt": self.toDate
}
}]
})
allFlows = dict((countryName, {})
for countryName in MediationConfig.getCountries())
currentConfig = dict(
(countryName, MediationConfig.getLobs(countryName))
for countryName in MediationConfig.getCountries())
newFlows = {}
newLobs = dict((countryName, {})
for countryName in MediationConfig.getCountries())
for doc in cursor:
for countryName, country in doc["data"].items():
for lobName, lob in country.items():
if lobName == "FOX":
continue
if currentConfig[countryName].get(lobName, None) == None:
newLobs[countryName][lobName] = True
for flowName, flow in lob.get("inputs", {}).items():
if flowName == "updatesCnt" or flowName == "sum":
continue
if flowName not in currentConfig[countryName].get(
lobName, {}).get("inputs", {}):
newFlows[createGlobalName(countryName, lobName,
flowName,
"inputs")] = True
for flowName, flow in lob.get("forwards", {}).items():
if flowName == "updatesCnt" or flowName == "sum":
continue
if flowName not in currentConfig[countryName].get(
lobName, {}).get("forwards", {}):
newFlows[createGlobalName(countryName, lobName,
flowName,
"forwards")] = True
# print(flowName)
insertedLobs = 0
for countryName, lobs in newLobs.items():
for lobName in lobs.keys():
insertedLobs += 1
MediationConfig.addLob(countryName, lobName)
for globalFlowName in newFlows.keys():
flow = globalNameToFlow(globalFlowName)
MediationConfig.addFlow(flow)
if insertedLobs > 0 or len(newFlows) > 0:
logging.info("inserted " + str(insertedLobs) + " lobs and " +
str(len(newFlows)) + " flows in " +
str(int(time.time() - start)) + " seconds")
def flowGET(country, lobName, flowName):
flow = MediationConfig.getLobWithCountry(country,
lobName)["flows"][flowName]
if shouldIncludeStatus():
status = FlowStatusManager().getLobDetailWithCountry(country,
lobName)[flowName]
flow["status"] = status
return jsonify(flow)
def getLobDetail(self, lobName):
lob = MediationConfig.getLob(lobName)
allStatuses = self.getAll(lob["country"])
lobFlowStatuses = {}
for flowName, flow in lob["flows"].items():
lobFlowStatuses[flowName] = allStatuses[flow["gName"]]
return lobFlowStatuses
def _enqueFlowsToAnalyze(self, flowQueue, country):
self.lastExecutions = self.statusManager.getAll(country)
countryLobs = MediationConfig.getLobs(country, enabledOnly=True)
for lobName, lob in countryLobs.items():
for flow in lob["flows"].values():
if self.shouldSchedule(flow):
job = {"flow": flow, "lastExecution": self.lastExecutions[flow["gName"]]}
flowQueue.put(job)
def _getFlowsToAnalyze(self, country):
flowsToAnalyze = []
lobsConfig = MediationConfig.getLobs(country, enabledOnly=True)
for lobName, lob in lobsConfig.items():
for flow in lob["flows"].values():
if self.shouldSchedule(flow):
flowsToAnalyze.append(flow)
return flowsToAnalyze
def getLob(country, lobName):
res = MediationConfig.getLobWithCountry(country, lobName)
if shouldIncludeStatus():
status = FlowStatusManager().getLobDetailWithCountry(country, lobName)
for k, v in status.items():
type = res["flows"][k]["type"]
res[type][k]["status"] = v
del res["flows"]
return jsonify(res)
def __init__(self, flow):
self.options = flow["options"]
self.independentDays = self.options.get("independentDays", [])
self.country = MediationConfig.getCountryByName(flow["country"])
self.HOLIDAYS = []
for holiday in self.country["holidays"]:
day, month = holiday.split(".")
self.HOLIDAYS.append((int(day), int(month)))
pass
def getCountry(country):
res = MediationConfig.getLobs(country)
for lob in res.values():
del lob["flows"]
del lob["inputs"]
del lob["forwards"]
if shouldIncludeStatus():
status = FlowStatusManager().getLobsOverview(country)
addStatus(res, status)
return jsonify(res)
def getAll(self, country):
lobs = MediationConfig.getLobs(country)
res = mongo.statuses().find_one({"_id": "lobs"}, {"_id": 0})
if res is None:
res = {}
statuses = {}
for lobName, lob in lobs.items():
for flowName, flow in lob["flows"].items():
gName = flow["gName"]
if flow["options"]["enabled"] is False:
statuses[gName] = {"status": status.DISABLED}
elif gName in res:
statuses[gName] = self._setStatusMetadata(res[gName], flow)
else:
statuses[gName] = {"status": status.NA}
return statuses
def __init__(self, date, flows, granularity=0):
self.flows = flows
self.flow = flows[0]
self.date = date
self.granularity = granularity
self.dates = SimilarPastDaysFinder(self.flow).findSimilarPastDays(date)
country = MediationConfig.getCountryByName(self.flow["country"])
self.options = self.flow["options"]
self.adjustment = None
lazyDayDifference = self.flow["options"].get("lazyDayDifference", None)
if lazyDayDifference != None:
for lazyDayStr in country["lazyDays"]:
lazyDay = util.stringToDate(lazyDayStr)
if lazyDay.date() == date.date():
self.adjustment = lazyDayDifference
break
def getCountriesConfig():
return jsonify(MediationConfig.getCountries())
def getEnabledCountry(country):
return jsonify(
{"enabled": MediationConfig.getCountryByName(country)["enabled"]})
def dataQueryV2():
"""
Endpoint for getting traffic data.
POST body:
{
"from":"01.02.2017",
"to":"15.02.2017",
"country":"CZ",
"lobName":"ACI",
"flowName":["GSM"],
"forwards":[],
"granularity":0
}
Response:
{
"data": [
{
"GSM": 188385532,
"_id": "2017-02-01T00:00:00+01:00",
"dayAverage": 1162595297.6666667,
"dayDifference": 1.023,
"expected": 161627916,
"status": "OK",
"tickDifference": 1.166
},
...
],
"metadata": {
"flowName": "GSM",
"granularity": 480,
"metrics": {
"GSM": {
"type": "traffic"
},
"dayAverage": {
"type": "traffic"
},
"dayDifference": {
"type": "difference"
},
"expected": {
"type": "traffic"
},
"status": {
"type": "other"
},
"tickDifference": {
"type": "difference"
}
}
}
"""
searchParam = request.get_json()
fromDate = util.stringToDate(searchParam["from"])
toDate = util.stringToDate(searchParam["to"])
country = searchParam["country"]
lobName = searchParam["lobName"]
lobConfig = MediationConfig.getLobWithCountry(country, lobName)
flows = []
granularity = searchParam.get("granularity", 0)
flows.append(lobConfig["flows"][searchParam["flowName"]])
response = {}
# Query the traffic data and add to metric list
mongoQuery = data_query.DateRangeGroupQuery(fromDate,
toDate,
flows,
granularity=granularity)
data = mongoQuery.execute()
metrics = {}
metricsList = []
flowName = mongoQuery.metrics[0]
metricsList.append(flowName)
metadata = mongoQuery.metadata
if len(flows) == 1:
# Run outage detection analysis
metric = metricsList[0]
flowLevelQuery = data_query.FlowLevelDateRangeQuery(
fromDate, toDate, flows, metadata["granularity"], data)
flowLevelData = flowLevelQuery.execute()
data = util.merge2DateLists(flowLevelData, None, data, None)
metricsList.extend(flowLevelQuery.metrics)
outageQuery = data_query.OutageDateRangeQuery(fromDate, toDate,
flows[0],
metadata["granularity"])
outageQuery.setPrecomputedData(data, metric)
outageList = outageQuery.execute()
data = util.merge2DateLists(outageList, [outageQuery.metric], data,
None)
metricsList.append(outageQuery.metric)
# Create metadata infor
for metric in metricsList:
if metric == flowName or metric == "dayAverage" or metric == "expected":
type = "traffic"
elif "Difference" in metric:
type = "difference"
else:
type = "other"
metrics[metric] = {"type": type}
response["data"] = data
response["metadata"] = {
**{
"metrics": metrics
},
**metadata, "flowName": flowName
}
return jsonify(response)
windowData = data[start:i + 1]
weights = [1 + x * x for x in range(0, len(windowData))]
s = 0
for x, y in zip(windowData, weights):
s += x * y
average = s / sum(weights)
output.append(average)
return output
fromDate = util.stringToTime("02.03.2017 00:00:00")
toDate = util.stringToTime("03.03.2017 00:00:00")
granularity = 15
q = DateRangeGroupQuery(
fromDate, toDate,
[MediationConfig.getLobWithCountry("CZ", "GSM")["flows"]["MSSBRN1A"]],
granularity)
# q = DateRangeGroupQuery(fromDate, toDate, [MediationConfig.getLobWithCountry("CZ","ACI")["flows"]["GSM"]], granularity)
lob1Data = util.dateDataListToList(q.execute(), "MSSBRN1A")
lob1Data.append(0)
lob1Data.append(0)
lob1Data.append(0)
lob1Data.append(0)
dates = list(
map(
lambda x: util.dateToTimeString(x).split("+")[0].split("T")[1].split(
":")[0], util.dateDataListToList(q.execute(), "_id")))
dataTicks = []
dataTickPos = []
for i in range(0, len(dates)):
if i % 4 == 0:
dataTicks = []
dataTickPos = []
for i in range(0, len(dates)):
if True:
dataTicks.append(dates[i])
dataTickPos.append(i)
return data1, expected, dataTicks, dataTickPos
fromDate = util.stringToDate("30.01.2017")
toDate = util.stringToDate("12.02.2017")
# lobName = "ICG"
# flowName = "CENTREX01"
lobName = "GSM"
flowName = "MSSBRN1B"
lob = MediationConfig.getLobWithCountry("CZ", lobName)
flow = lob["flows"][flowName]
flows = [flow]
granularity = 240
data = []
expected = []
ticks = []
pos = []
d, e, t, p = getData(flows, util.stringToDate("28.09.2016"),
util.stringToDate("29.09.2016"))
data += d
expected += e
ticks += t
posLen = len(pos)
pos += [i + posLen for i in p]
def getFlowOptions(country, lobName, flowName):
lobConfig = MediationConfig.getLobWithCountry(country, lobName)
flowConfig = lobConfig["flows"][flowName]
return jsonify(flowConfig)
def deleteFlow(country, lobName, flowName):
lob = MediationConfig.getLobWithCountry(country, lobName)
if flowName in lob["flows"]:
MediationConfig.deleteFlow(lob["flows"][flowName])
return getLob(country, lobName)
0.796042342418818, 0.9346540748162848, 0.7939754809772508, 0.962470501562014,
0.9929573860362824, 1]},
{"lobName": 'EWG',
"data": [0.9761478362504887, 0.9583614156352167, 1.2336652720530112, 1.1580357149388305, 1.090568389327002,
0.9600084918076629, 0.9568424857376663, 0.8256895883325289, 0.5914861789815206,
1.0121511182076122, 0.8653533492971078, 0.5704172842604073, 0.538653445993039, 0.5164638232491638,
0.4858079169661382, 0.41779533433377064, 0.7993573176963311, 0.9092267083181577,
1.007730199059085, 1.0702191864202208, 0.9082408024540793, 1]},
{"lobName": 'TPP', "data": [0.9994509893629189, 0.9329863891112891, 0.8681779342746304, 0.8697074230973179,
0.8752423267949921, 0.8735428814090898, 0.872957061838658, 0.8761341580186848,
0.8998970751542529, 0.8653601909437505, 0.8668891761965745, 0.7537979083127982,
0.9302389942504077, 0.9967766261759137, 0.9968273519280757, 0.9897328514978554,
0.8680083180866294, 0.9732343917814772, 0.9873704480552615, 0.9832499869540516,
0.9831210955086642, 1]}]
CZ_LOBS = MediationConfig.getCountryByName("CZ")
# CZ_LOBS = ["SMS", "GSM", "MMS"]
def getnormalizedFlowLevel(lobName, granularity=0):
fromDate = util.jsStringToDate("2016-12-15T00:00:00.000Z").replace(hour=0, minute=0, second=0)
toDate = util.jsStringToDate("2017-01-05T00:00:00.000Z").replace(hour=0, minute=0, second=0)
response = {}
mongoQuery = data_query.DateRangeGroupQuery(fromDate, toDate, [lobName], granularity)
data = mongoQuery.execute()
metrics = {}
metricsList = mongoQuery.metrics
metadata = mongoQuery.metadata
data = data_query.medianDateRange(fromDate, toDate, lobName, granularity, data)
return util.dateDataListToList(data, "normalizedFlowLevel")
def resetFlowStatus(country, lobName, flowName):
flow = MediationConfig.getFlow(country, lobName, flowName)
FlowStatusManager().resetStatus(flow)
return flowGET(country, lobName, flowName)
