def __init__(self, flow, dates, granularity):
super().__init__()
self.flow = flow
self.coll = mongo.traffic()
self.dates = dates
self.granularity = granularity
self.dataPath = "data." + self.flow["dataPath"]
self.metric = self.flow["name"]
def _executeInternal(self):
start = time.time()
cursor = mongo.traffic().find({
"$and": [{
"_id": {
"$gte": self.fromDate
}
}, {
"_id": {
"$lt": self.toDate
}
}]
})
allFlows = dict((countryName, {})
for countryName in MediationConfig.getCountries())
currentConfig = dict(
(countryName, MediationConfig.getLobs(countryName))
for countryName in MediationConfig.getCountries())
newFlows = {}
newLobs = dict((countryName, {})
for countryName in MediationConfig.getCountries())
for doc in cursor:
for countryName, country in doc["data"].items():
for lobName, lob in country.items():
if lobName == "FOX":
continue
if currentConfig[countryName].get(lobName, None) == None:
newLobs[countryName][lobName] = True
for flowName, flow in lob.get("inputs", {}).items():
if flowName == "updatesCnt" or flowName == "sum":
continue
if flowName not in currentConfig[countryName].get(
lobName, {}).get("inputs", {}):
newFlows[createGlobalName(countryName, lobName,
flowName,
"inputs")] = True
for flowName, flow in lob.get("forwards", {}).items():
if flowName == "updatesCnt" or flowName == "sum":
continue
if flowName not in currentConfig[countryName].get(
lobName, {}).get("forwards", {}):
newFlows[createGlobalName(countryName, lobName,
flowName,
"forwards")] = True
# print(flowName)
insertedLobs = 0
for countryName, lobs in newLobs.items():
for lobName in lobs.keys():
insertedLobs += 1
MediationConfig.addLob(countryName, lobName)
for globalFlowName in newFlows.keys():
flow = globalNameToFlow(globalFlowName)
MediationConfig.addFlow(flow)
if insertedLobs > 0 or len(newFlows) > 0:
logging.info("inserted " + str(insertedLobs) + " lobs and " +
str(len(newFlows)) + " flows in " +
str(int(time.time() - start)) + " seconds")
def __init__(self, dates, flows, granularity=0):
self.query = None
self.coll = mongo.traffic()
self.metrics = []
self.dataPaths = []
self.dates = [util.resetDateTimeMidnight(d) for d in dates]
self.flows = flows
self.granularity = int(granularity)
self.createDataPathAndOutputs2()
self.metadata = {}
def __init__(self, dates, flows, granularity=0):
self.query = None
self.coll = mongo.traffic()
self.flow = flows[0]
self.metrics = []
self.dates = [util.resetDateTimeMidnight(d) for d in dates]
self.metric = self.flow["name"]
self.metrics.append(self.metric)
self.granularity = int(granularity)
self.metadata = {}
self.determinateGranularity()
def saveMediationData(self, row):
"""
:param data:
dict{"country":"CZ",
"lobName":"ACI",
"type":"inputs",
"flowName":"GSM",
"dataSize":497464,
"time":"01.01.2017 00:00:24"}
:return:
"""
time = util.stringToTime(row["time"]).replace(second=0)
updatePath = "data." + row["country"] + "." + row[
"lobName"] + "." + row["type"] + "."
try:
dataSize = int(row["dataSize"])
except ValueError:
print("ValueError: " + row)
dataSize = 0
updateObj = {
"$inc": {
updatePath + row["flowName"]: dataSize,
updatePath + "sum": dataSize,
updatePath + "updatesCnt": 1
}
}
try:
mongo.traffic().update({'_id': time}, updateObj, upsert=True)
logging.info("Inserted row for " + row["lobName"] + " " +
row["flowName"] + " at " + str(time) + " of size " +
str(dataSize))
return True
except Exception as e:
logging.error("Error while persisting data " + str(row) +
" exception: " + str(e))
return False
def insertRows(self, rowList):
coll = mongo.traffic()
updates = _sumUpdates(list(map(_createRowUpdateDict, rowList)))
print("Inserted " + str(len(rowList)) + " rows")
for key, value in updates.items():
coll.update({'_id': key}, value, upsert=True)