def getLobsOverview(self, country): allStatuses = self.getAll(country) lobStatusDict = {} for lobName, lob in MediationConfig.getLobs(country).items(): ok = 0 warning = 0 outage = 0 expired = 0 disabled = 0 for flow in lob["flows"].values(): gName = flow["gName"] if flow["options"]["enabled"] is False: disabled += 1 continue flowStatus = allStatuses.get(gName, {"status": status.NA})["status"] if flowStatus == status.NA: expired += 1 elif flowStatus == status.OK: ok += 1 elif flowStatus == status.WARNING: warning += 1 elif flowStatus == status.OUTAGE: outage += 1 lobStatusDict[lobName] = { status.OK: ok, status.WARNING: warning, status.OUTAGE: outage, status.NA: expired, status.DISABLED: disabled } return lobStatusDict
def _getFlowsToAnalyze(self, country): flowsToAnalyze = [] lobsConfig = MediationConfig.getLobs(country, enabledOnly=True) for lobName, lob in lobsConfig.items(): for flow in lob["flows"].values(): if self.shouldSchedule(flow): flowsToAnalyze.append(flow) return flowsToAnalyze
def _enqueFlowsToAnalyze(self, flowQueue, country): self.lastExecutions = self.statusManager.getAll(country) countryLobs = MediationConfig.getLobs(country, enabledOnly=True) for lobName, lob in countryLobs.items(): for flow in lob["flows"].values(): if self.shouldSchedule(flow): job = {"flow": flow, "lastExecution": self.lastExecutions[flow["gName"]]} flowQueue.put(job)
def _executeInternal(self): start = time.time() cursor = mongo.traffic().find({ "$and": [{ "_id": { "$gte": self.fromDate } }, { "_id": { "$lt": self.toDate } }] }) allFlows = dict((countryName, {}) for countryName in MediationConfig.getCountries()) currentConfig = dict( (countryName, MediationConfig.getLobs(countryName)) for countryName in MediationConfig.getCountries()) newFlows = {} newLobs = dict((countryName, {}) for countryName in MediationConfig.getCountries()) for doc in cursor: for countryName, country in doc["data"].items(): for lobName, lob in country.items(): if lobName == "FOX": continue if currentConfig[countryName].get(lobName, None) == None: newLobs[countryName][lobName] = True for flowName, flow in lob.get("inputs", {}).items(): if flowName == "updatesCnt" or flowName == "sum": continue if flowName not in currentConfig[countryName].get( lobName, {}).get("inputs", {}): newFlows[createGlobalName(countryName, lobName, flowName, "inputs")] = True for flowName, flow in lob.get("forwards", {}).items(): if flowName == "updatesCnt" or flowName == "sum": continue if flowName not in currentConfig[countryName].get( lobName, {}).get("forwards", {}): newFlows[createGlobalName(countryName, lobName, flowName, "forwards")] = True # print(flowName) insertedLobs = 0 for countryName, lobs in newLobs.items(): for lobName in lobs.keys(): insertedLobs += 1 MediationConfig.addLob(countryName, lobName) for globalFlowName in newFlows.keys(): flow = globalNameToFlow(globalFlowName) MediationConfig.addFlow(flow) if insertedLobs > 0 or len(newFlows) > 0: logging.info("inserted " + str(insertedLobs) + " lobs and " + str(len(newFlows)) + " flows in " + str(int(time.time() - start)) + " seconds")
def getCountry(country): res = MediationConfig.getLobs(country) for lob in res.values(): del lob["flows"] del lob["inputs"] del lob["forwards"] if shouldIncludeStatus(): status = FlowStatusManager().getLobsOverview(country) addStatus(res, status) return jsonify(res)
def getAll(self, country): lobs = MediationConfig.getLobs(country) res = mongo.statuses().find_one({"_id": "lobs"}, {"_id": 0}) if res is None: res = {} statuses = {} for lobName, lob in lobs.items(): for flowName, flow in lob["flows"].items(): gName = flow["gName"] if flow["options"]["enabled"] is False: statuses[gName] = {"status": status.DISABLED} elif gName in res: statuses[gName] = self._setStatusMetadata(res[gName], flow) else: statuses[gName] = {"status": status.NA} return statuses
from mediation.flow_analyzer import FlowAnalyzer queue = Queue() class Worker(threading.Thread): def __init__(self, queue): super().__init__() self.queue = queue def run(self): try: while True: job = self.queue.get_nowait() FlowAnalyzer(job).run(AppConfig.getCurrentTime()) except Exception as e: pass for lob in MediationConfig.getLobs("CZ", enabledOnly=True).values(): for flow in lob["flows"].values(): queue.put(flow) workers = [Worker(queue) for i in range(0, 8)] for worker in workers: worker.start() for worker in workers: worker.join() print("finished")