def test_temperature_distribution(self): for p in self.params: with self.subTest(): self.assertEqual( Climate.calculate_temperature_distribution(temps=p), Climate.calculate_distribution( data=p, limits=Climate.TEMP_DISTRIBUTION_LIMITS))
def create_daily_statistics(fromDate, toDate, siteId, limitInMins=3): """ | Create daily stat :param fromDate: start date of calculation :param toDate: end date of calculation :param siteId: site :param limitInMins: limit before calculation :return: None """ limit = datetime.timedelta(minutes=(limitInMins)) fromDate = fromDate.replace(hour=0, minute=0, second=0) delta = toDate - fromDate existing = DailyStatistics.objects.filter( year__range=(fromDate.year, toDate.year), siteId=siteId).values() existing_dates = [] for i in existing: existing_dates.append(datetime.date(year=i.get('year'), month=i.get('month'), day=i.get('day'))) daily_data = [] for i in range(delta.days + 1): f = fromDate + datetime.timedelta(days=i) t = fromDate + datetime.timedelta(days=i + 1) rawDataSet = RawData.objects.filter(siteId=siteId, createdDate__range=(f, t)) manualDataSet = RawManualData.objects.filter( siteId=siteId, year=f.year, month=f.month, day=f.day) precipitation = None if rawDataSet.count() or manualDataSet.count(): d = DailyStatisticsDTO(year=f.year, month=f.month, day=f.day, siteId=siteId) d.existing = datetime.date(year=f.year, month=f.month, day=f.day) in existing_dates daily_data.append(d) if rawDataSet.count(): d = DailyStatisticsDTO(dataAvailable=rawDataSet.count(), siteId=siteId) temps = [] rhs = [] winds = [] last_timestamp = None for j in rawDataSet: current_timestamp = j.createdDate if not last_timestamp or (current_timestamp - last_timestamp) >= limit: if j.temperature is not None: temps.append(j.temperature) if j.humidity is not None: rhs.append(j.humidity) if j.precipitation is not None: if precipitation is None: precipitation = decimal.Decimal(0.0) else: precipitation = precipitation + j.precipitation if j.windDir is not None: winds.append(j.windDir) last_timestamp = current_timestamp tempDistribution = Climate.calculate_temperature_distribution(temps) d.tempDistribution = ''.join(str(e) + ',' for e in tempDistribution)[:-1] rhDistribution = Climate.calculate_rh_distribution(rhs) d.rhDistribution = ''.join(str(e) + ',' for e in rhDistribution)[:-1] windDistribution = Climate.calculate_wind_distribution(winds) d.windDistribution = ''.join(str(e) + ',' for e in windDistribution)[:-1] if len(temps) > 0: d.tempMin = min(temps) d.tempMax = max(temps) d.tempAvg = sum(temps) / len(temps) if precipitation is not None: d.precipitation = precipitation if manualDataSet.count(): if manualDataSet[0].tMin is not None: d.tempMin = manualDataSet[0].tMin if manualDataSet[0].tMax is not None: d.tempMax = manualDataSet[0].tMax if manualDataSet[0].precAmount is not None: d.precipitation = manualDataSet[0].precAmount DailyStatistics.objects.bulk_create( DailyStatistics( year=d.year, month=d.month, day=d.day, siteId=d.siteId, dataAvailable=d.dataAvailable or 0, tempMin=d.tempMin, tempMax=d.tempMax, tempAvg=d.tempAvg, precipitation=d.precipitation, tempDistribution=d.tempDistribution or '', rhDistribution=d.rhDistribution or '', windDistribution=d.windDistribution or '' ) for d in daily_data if not d.existing ) with transaction.atomic(): for d in daily_data: if d.existing: DailyStatistics.objects.filter(siteId=d.siteId, year=d.year, month=d.month, day=d.day).update( dataAvailable=d.dataAvailable or 0, tempMin=d.tempMin, tempMax=d.tempMax, tempAvg=d.tempAvg, precipitation=d.precipitation, tempDistribution=d.tempDistribution or '', rhDistribution=d.rhDistribution or '', windDistribution=d.windDistribution or '' ) logger.info("daily stat finished")
def create_monthly_statistics(fromDate, toDate, siteId): """ | Create monthly stat :param fromDate: start date of calculation :param toDate: end date of the calculation :param siteId: site :return: None """ logger = logging.getLogger(__name__) logger.error("create monthly stat from {} to {}".format(fromDate, toDate)) fromDate = fromDate.replace(hour=0, minute=0, second=0, day=1) if (toDate.month < 12): toDate = toDate.replace(month=toDate.month + 1, day=1, hour=0, minute=0, second=0) else: toDate = toDate.replace(year=toDate.year + 1, month=1, day=1, hour=0, minute=0, second=0) f = fromDate while f < toDate: rawDataSet = DailyStatistics.objects.filter(year=f.year, month=f.month).filter(siteId=siteId) if rawDataSet.count(): d, created = MonthlyStatistics.objects.update_or_create(siteId=siteId, month=f.month, year=f.year) d.dataAvailable = rawDataSet.count() tempmins = [] tempmaxs = [] tempavgs = [] precipitation = decimal.Decimal(0.0) for j in rawDataSet: if j.tempMin is not None: tempmins.append(j.tempMin) if j.tempMax is not None: tempmaxs.append(j.tempMax) if j.tempAvg is not None: tempavgs.append(j.tempAvg) if j.precipitation is not None: precipitation = precipitation + j.precipitation d.dataAvailable = rawDataSet.count() if len(tempmins) > 0: d.tempMin = min(tempmins) d.tempMinAvg = sum(tempmins) / len(tempmins) if len(tempmaxs) > 0: d.tempMax = max(tempmaxs) d.tempMaxAvg = sum(tempmaxs) / len(tempmaxs) if len(tempavgs) > 0: d.tempAvg = sum(tempavgs) / len(tempavgs) start_of_current_month = datetime.datetime( year=f.year, month=f.month, day=1, hour=0, minute=0) start_of_next_month = start_of_current_month + relativedelta(months=1) rawDataSet = RawData.objects\ .filter(createdDate__range=(start_of_current_month, start_of_next_month))\ .filter(siteId=siteId) temps = [] rhs = [] winds = [] for j in rawDataSet: if j.temperature is not None: temps.append(j.temperature) if j.humidity is not None: rhs.append(j.humidity) if j.windDir is not None: winds.append(j.windDir) tempDistribution = Climate.calculate_temperature_distribution(temps) d.tempDistribution = ''.join(str(e) + ',' for e in tempDistribution)[:-1] rhDistribution = Climate.calculate_rh_distribution(rhs) d.rhDistribution = ''.join(str(e) + ',' for e in rhDistribution)[:-1] windDistribution = Climate.calculate_wind_distribution(winds) d.windDistribution = ''.join(str(e) + ',' for e in windDistribution)[:-1] d.precipitation = precipitation d.summerDays = Climate.get_nr_summer_days(tempmaxs) d.frostDays = Climate.get_nr_frost_days(tempmins) d.winterDays = Climate.get_nr_winter_days(tempmaxs) d.coldDays = Climate.get_nr_cold_days(tempmins) d.warmNights = Climate.get_nr_warm_nights(tempmins) d.warmDays = Climate.get_nr_warm_days(tempmaxs) d.hotDays = Climate.get_nr_hot_days(tempmaxs) manualDataSet = RawManualData.objects.filter(siteId=siteId).filter(year=f.year).filter(month=f.month) significants = {} for day in manualDataSet: significants = Climate.count_significants(significants, day.weatherCode) d.significants = significants d.save() if (f.month == 12): f = f.replace(year=f.year + 1, month=1) else: f = f.replace(month=f.month + 1)