def _getLatestCompleteTicTime(self, time):
granularity = self.granularity
minuteOfDay = time.hour * 60 + time.minute
minutesOverInterval = minuteOfDay % granularity
if granularity == 60:
latestClosedIntervalTime = time - datetime.timedelta(minutes=minutesOverInterval + granularity)
latestClosedIntervalTime = latestClosedIntervalTime.replace(second=0, microsecond=0)
return util.getTicTime(latestClosedIntervalTime.astimezone(AppConfig.getTimezone()), granularity)
else:
naiveTime = time.replace(tzinfo=None)
latestClosedIntervalNaiveTime = naiveTime - datetime.timedelta(minutes=minutesOverInterval + granularity)
latestClosedIntervalNaiveTime = latestClosedIntervalNaiveTime.replace(second=0, microsecond=0)
localized = AppConfig.getTimezone().localize(latestClosedIntervalNaiveTime).astimezone(AppConfig.getTimezone())
return util.getTicTime(localized, granularity)
def execute(self):
self.prepare()
result = list(self.coll.aggregate(self.query))
resultDict = {}
for i in result:
group = i["_id"]
date = datetime.datetime(group["year"], group["month"],
group["dayOfMonth"], int(group["hour"]),
int(group["minute"]))
from common import AppConfig
date = AppConfig.getTimezone().localize(date)
i["_id"] = date
resultDict[date] = i
granularityDelta = datetime.timedelta(minutes=self.granularity)
nullObject = {}
for metric in self.metrics:
nullObject[metric] = 0
for date in self.dates:
d = date
while d < date + datetime.timedelta(days=1):
if d not in resultDict: # TODO: check if some results can have just few of the metrics.
resultDict[d] = {**nullObject, **{"_id": d}}
d += granularityDelta
result = sorted(resultDict.values(), key=lambda x: x["_id"])
return result
def getTicTime(time, granularity):
minuteOfDay = time.hour * 60 + time.minute
minutesOverInterval = minuteOfDay % granularity
naiveTime = time.replace(tzinfo=None)
latestClosedIntervalNaiveTime = naiveTime - datetime.timedelta(
minutes=minutesOverInterval)
latestClosedIntervalNaiveTime = latestClosedIntervalNaiveTime.replace(
second=0, microsecond=0)
return AppConfig.getTimezone().localize(latestClosedIntervalNaiveTime)
def execute(self):
timezone = AppConfig.getTimezone()
def convertToTimezone(x):
x["_id"] = x["anyDate"].astimezone(timezone)
return x
res = list(map(convertToTimezone, list(self._executeQuery())))
resultDict = self.aggregate(res)
return resultDict
def _executeMongoAggregateQuery(self):
result = list(self.coll.aggregate(self.query))
resultDict = {}
from common import AppConfig
appTimezone = AppConfig.getTimezone()
for i in result:
group = i["_id"]
utcDate = datetime.datetime(group["year"],
group["month"], group["dayOfMonth"],
int(group["hour"]),
int(group["minute"]), 0, 0, utc)
date = utcDate.astimezone(appTimezone)
anyDate = i["anyDate"].astimezone(appTimezone)
i["_id"] = date
resultDict[date] = i
return resultDict
def getNextTic(d, granularity):
"""
returns next tick
:param d:
:param granularity:
:return:
"""
prevOffsetSeconds = d.tzinfo._utcoffset.total_seconds()
from common import AppConfig
newTic = (d + datetime.timedelta(minutes=granularity)).astimezone(
AppConfig.getTimezone())
newOffsetSeconds = newTic.tzinfo._utcoffset.total_seconds()
if prevOffsetSeconds != newOffsetSeconds:
if prevOffsetSeconds < newOffsetSeconds:
newTic = getTicTime(newTic, granularity)
else:
newTic = roundToNextTicTime(newTic, granularity)
return getTicTime(newTic, granularity)
resulttic["expected"] = tic["expected"]
resulttic["dayAverage"] = tic["dayAverage"]
resultData.append(resulttic)
self.metrics = [
"tickDifference", "dayDifference", "expected", "dayAverage"
]
return resultData
if __name__ == "__main__":
gran = 120
flow = {
'lobName': 'ACI',
'dataPath': 'CZ.ACI.inputs.GSM',
'country': 'CZ',
'gName': 'CZ_ACI_GSM',
'name': 'GSM',
'options': {
'softAlarmLevel': 0.75,
'hardAlarmLevel': 0.51,
'minimalExpectation': 1,
'enabled': True,
'difference': 'day',
'granularity': 480
},
'type': 'inputs'
}
dates = [
AppConfig.getTimezone().localize(datetime.datetime(2017, 3, 26, 0, 0))
]
FlowLevelDateRangeQuery(flow, dates, gran).execute()
def jsStringToDate(string):
return AppConfig.getTimezone().localize(dateutil.parser.parse(string))
def stringToDate(dateTimeStr):
return AppConfig.getTimezone().localize(
datetime.strptime(dateTimeStr, "%d.%m.%y %H:%M:%S"))
def getNextDay(d):
dayDelta = datetime.timedelta(days=1)
return AppConfig.getTimezone().localize(d.replace(tzinfo=None) + dayDelta)
import csv
import pytz
import mediation.data_receiver.DataReceiverConfig as config
import mediation.data_receiver.DataReceiverUtil as util
from common import AppConfig
from .data_insertor import DataInsertor
LATEST_DATE = util.stringToDate("20.02.16 00:00:00").replace(tzinfo=AppConfig.getTimezone())
def isValidFlow(flow):
return flow["date"] > LATEST_DATE and flow["country"] in config.COUNTRIES and flow["lob"] not in config.IGNORE_LOBS
"""
deprecated
"""
class FileParser:
def __init__(self):
self.batchSize = 100000
def parseInputs(self, inputFile):
inputsList = []
dataInsertor = DataInsertor()
with open(inputFile, 'r') as csvfile:
spamreader = csv.reader(csvfile, delimiter=';', quotechar='"')
for row in spamreader:
try:
input = self.createInputRow(row)
if isValidFlow(input):