Esempio n. 1
0
def process_household(household, bounds):
    """
    Processes each household time series, first by regularizing step
    of the raw series to the fifteen minutes time series, then by
    aggregating fifteen minutes series to hourly, daily, monthly.
    """
    print "Processing household: %s"%(household.alt_codes.all()[0],)
    for variable in (VAR_PERIOD, VAR_ENERGY_PERIOD):
        raw_series_db = household.timeseries.filter(time_step__isnull=True,
                variable__id = {VAR_PERIOD: VAR_CUMULATIVE,
                    VAR_ENERGY_PERIOD: VAR_ENERGY_CUMULATIVE}[variable])[:1]
        if not raw_series_db:
            continue
        raw_series_db = raw_series_db[0]
        fifteen_min_series_db = household.timeseries.get(time_step__id=
                TSTEP_FIFTEEN_MINUTES, variable__id=variable)
        s1, e1 = timeseries_bounding_dates_from_db(db.connection,
                raw_series_db.id)
        s2, e2 = timeseries_bounding_dates_from_db(db.connection,
                fifteen_min_series_db.id)
    #    if e2 and (e1-e2).seconds<15*60:
    #        return
        fifteen_min_series = regularize_raw_series(raw_series_db,
                fifteen_min_series_db, s1,e1,s2,e2)
        if fifteen_min_series.bounding_dates():
            bounds[variable]['fifteen_start'] = min(
                    bounds[variable]['fifteen_start'],
                    fifteen_min_series.bounding_dates()[0])
            bounds[variable]['fifteen_end'] = max(
                    bounds[variable]['fifteen_end'],
                    fifteen_min_series.bounding_dates()[1])
        result = fifteen_min_series
        monthly_series = None
        for time_step_id in (TSTEP_HOURLY, TSTEP_DAILY, TSTEP_MONTHLY):
            if not result:
                break
            result = aggregate_household_series(bounds, household,
                    result, time_step_id, variable)
            if time_step_id == TSTEP_MONTHLY:
                monthly_series = result
        if not monthly_series:
            return
        # Cost calculation only if monthly_series present
        cost_timeseries_db = household.timeseries.get(time_step__id =
                TSTEP_MONTHLY, variable__id = 
                {VAR_PERIOD: VAR_COST,
                 VAR_ENERGY_PERIOD: VAR_ENERGY_COST
                }[variable])
        cost_timeseries = calculate_cost_timeseries(monthly_series,
                rate = {
                    VAR_PERIOD: CUBIC_METER_FLAT_RATE,
                    VAR_ENERGY_PERIOD: KWH_FLAT_RATE}[variable])
        cost_timeseries.id = cost_timeseries_db.id
        cost_timeseries.write_to_db(db = db.connection, commit=True)
Esempio n. 2
0
def MultiTimeseriesProcessDb(method, timeseries_arg, out_timeseries_id,
                             db, read_tstep_func, transaction=None, 
                             commit=True, options={}):
    out_timeseries = Timeseries(id = out_timeseries_id)
    opts = copy.deepcopy(options)
    if 'append_only' in opts and opts['append_only']:
        bounds = timeseries_bounding_dates_from_db(db, 
                                                   id = out_timeseries_id)
        opts['start_date'] = bounds[1] if bounds else None;
        opts['interval_exclusive'] = True
    tseries_arg={}
    for key in timeseries_arg:
        ts = Timeseries(id=timeseries_arg[key])
        if ('append_only' in opts and opts['append_only']) \
                         and opts['start_date'] is not None:
            ts.read_from_db(db, bottom_only=True)
            if ts.bounding_dates()[0]>opts['start_date']:
                ts.read_from_db(db)
        else:
            ts.read_from_db(db)
        ts.time_step = read_tstep_func(ts.id)
        tseries_arg[key] = ts
    MultiTimeseriesProcess(method, tseries_arg, out_timeseries, opts)
    if 'append_only' in opts and opts['append_only']:
        out_timeseries.append_to_db(db=db, transaction=transaction,
                                    commit=commit)
    else:
        out_timeseries.write_to_db(db=db, transaction=transaction, 
                                   commit=commit)
Esempio n. 3
0
 def handle(self, *args, **options):
     try:
         prefix = args[0]
     except IndexError:
         prefix = ""
     try:
         if prefix:
             users = User.objects.filter(username__icontains=prefix)
         else:
             users = User.objects.all()
         out = []
         print "output for {x} users".format(x=len(users))
         for user in users:
             print "looking for user %s" % user.username
             try:
                 household = Household.objects.get(user=user)
                 for variable in (VAR_PERIOD, VAR_ENERGY_PERIOD):
                     if variable == VAR_PERIOD:
                         _type = "WATER"
                     else:
                         _type = "ENERGY"
                     raw_series_db = household.timeseries.filter(
                         time_step__isnull=True,
                         variable__id={
                             VAR_PERIOD: VAR_CUMULATIVE,
                             VAR_ENERGY_PERIOD: VAR_ENERGY_CUMULATIVE
                         }[variable])[:1]
                     if not raw_series_db:
                         continue
                     raw_series_db = raw_series_db[0]
                     s1, e1 = timeseries_bounding_dates_from_db(
                         db.connection, raw_series_db.id)
                     if s1:
                         start = s1.strftime("%Y-%m-%d %H:%M:%S")
                     else:
                         start = ""
                     if e1:
                         end = e1.strftime("%Y-%m-%d %H:%M:%S")
                     else:
                         end = ""
                     out.append([user.username, start, end, _type])
             except Exception as e:
                 print (repr(e))
                 continue
         import time
         _outfile = "%s_bounding_dates.csv" % prefix
         _path = "data/"
         with open(path.join(_path, _outfile), 'w') as of:
             a = csv.writer(of, delimiter=',',
                            quotechar='"',
                            quoting=csv.QUOTE_ALL)
             a.writerows(out)
     except Exception as e:
         raise CommandError(repr(e))
Esempio n. 4
0
def parse_and_save_timeseries(device_id, timeseries_id):
    """
    Reads a RAW timeseries from REST API and saves in our local
    database using the timeseries_id.
    ``device_id`` will be the ``identifier`` used in other functions,
    usualy is the customerID==deviceID
    """
    s, e = timeseries_bounding_dates_from_db(db.connection, timeseries_id)
    if s or e:
        print 'Raw timeseries id=%s has already data, skipping...' % (
            timeseries_id, )
        return
    timeseries = TSeries()
    timeseries.id = timeseries_id
    for timestamp, value in ibm_restapi.get_raw_timeseries(device_id):
        timeseries[timestamp] = value
    timeseries.write_to_db(db=db.connection,
                           transaction=transaction,
                           commit=False)
Esempio n. 5
0
def create_objects(dma, household_identifier, series, force=False):
    """
    When a household is fully parsed then this command is called to create
    database objects thus: user (household owner), household, database time
    series placeholders (for raw data and for processed data), to write actual
    time series data in database and finally to estimate the household
    occupancy.
    """
    print "Processing household %s, user username will be %s as well"%(
            household_identifier, household_identifier)
    # Create user (household owner), household, database series placeholders
    user = create_user(household_identifier)
    household=create_household(household_identifier, user,
            zone=dma.id)
    db_series = create_raw_timeseries(household)
    create_processed_timeseries(household)
    timeseries_data = {}
    # Now we will create timeseries.Timeseries() and we will add
    # parsed values
    for variable in db_series:
        if variable not in ('WaterCold', 'Electricity'):
            continue
        s, e = timeseries_bounding_dates_from_db(db.connection,
                db_series[variable].id)
        if not force and (s or e):
            print 'Raw timeseries id=%s has already data, skipping...'%(
                    db_series[variable].id,)
            continue
        timeseries = TSeries()
        timeseries.id = db_series[variable].id
        total = 0.0
        for timestamp, value in series[variable]:
            if not math.isnan(value):
                total += value
                timeseries[timestamp] = total
            else:
                timeseries[timestamp] = float('NaN')
        timeseries_data[variable] = timeseries
        timeseries.write_to_db(db=db.connection,
                transaction=transaction,
                commit=False)
    if 'WaterCold' in timeseries_data:
        calc_occupancy(timeseries_data['WaterCold'], household)
Esempio n. 6
0
def parse_and_save_timeseries(device_id, timeseries_id):
    """
    Reads a RAW timeseries from REST API and saves in our local
    database using the timeseries_id.
    ``device_id`` will be the ``identifier`` used in other functions,
    usualy is the customerID==deviceID
    """
    s, e = timeseries_bounding_dates_from_db(db.connection,
            timeseries_id)
    if s or e:
        print 'Raw timeseries id=%s has already data, skipping...'%(
                timeseries_id,)
        return
    timeseries = TSeries()
    timeseries.id = timeseries_id
    for timestamp, value in ibm_restapi.get_raw_timeseries(device_id):
        timeseries[timestamp] = value
    timeseries.write_to_db(db=db.connection,
            transaction=transaction,
            commit=False)
Esempio n. 7
0
def InterpolateDbTimeseries(source_id, dest_id, curve_type, curve_data,
                            db, data_columns=(0,1), logarithmic=False,
                            offset=0, append_only=False,
                            transaction=None, commit=True):
    if append_only:
        bounds = timeseries_bounding_dates_from_db(db, id = dest_id)
        start_date = bounds[1] if bounds else None;
    ts = Timeseries(id=source_id)
    if append_only and start_date is not None:
        ts.read_from_db(db, bottom_only=True)
        if ts.bounding_dates()[0]>start_date:
            ts.read_from_db(db)
        while ts.bounding_dates()[0]<=start_date:
            del(ts[ts.bounding_dates()[0]])
            if len(ts)==0: return
    else:
        ts.read_from_db(db)
    curve_list = TransientCurveList()
    if curve_type=='SingleCurve':
        curve_list.add(logarithmic=logarithmic, 
                       offset=CurvePoint(offset, 0))
        super(TransientCurve, 
              curve_list[0]).read_fp(StringIO(curve_data),
                                                data_columns)
    elif curve_type=='StageDischargeMulti':
        curve_list.read_fp(StringIO(curve_data))
    else:
        assert(False)
    out_timeseries = curve_list.interpolate_ts(ts)
    out_timeseries.id = dest_id
    if append_only:
        out_timeseries.append_to_db(db=db, transaction=transaction,
                                    commit=commit)
    else:
        out_timeseries.write_to_db(db=db, transaction=transaction, 
                                   commit=commit)
Esempio n. 8
0
def AggregateDbTimeseries(source_id, dest_id, db, read_tstep_func, transaction=None,
                          commit=True, missing_allowed=0.0,
                          missing_flag='MISSING', append_only=False,
                          last_incomplete=False, all_incomplete=False):
    source = Timeseries(id=source_id, time_step=read_tstep_func(source_id))
    dest_step = read_tstep_func(dest_id)
    if append_only:
        bounds = timeseries_bounding_dates_from_db(db = db, id = dest_id)
        end_date = bounds[1] if bounds else None
    source.read_from_db(db)
    dest = source.aggregate(target_step=dest_step,
                            missing_allowed=missing_allowed, 
                            missing_flag=missing_flag,
                            last_incomplete=last_incomplete,
                            all_incomplete=all_incomplete)[0]
    dest.id = dest_id
    if append_only:
        d=dest.bounding_dates()
        while (d is not None) and (end_date is not None) and d[0]<=end_date:
            del dest[d[0]]
            d=dest.bounding_dates()
        dest.append_to_db(db=db, transaction=transaction, commit=commit)
    else:
        dest.write_to_db(db=db, transaction=transaction, commit=commit)
Esempio n. 9
0
def process_household(household):
    for variable in (VAR_PERIOD, VAR_ENERGY_PERIOD):
        raw_series_db = household \
            .timeseries \
            .filter(time_step__isnull=True,
                    variable__id={
                        VAR_PERIOD: VAR_CUMULATIVE,
                        VAR_ENERGY_PERIOD: VAR_ENERGY_CUMULATIVE
                    }[variable])[:1]
        if not raw_series_db:
            continue
        raw_series_db = raw_series_db[0]
        fifteen_min_series_db = household \
            .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                            variable__id=variable)
        s1, e1 = timeseries_bounding_dates_from_db(db.connection,
                                                   raw_series_db.id)
        s2, e2 = timeseries_bounding_dates_from_db(db.connection,
                                                   fifteen_min_series_db.id)
        #    if e2 and (e1-e2).seconds<15*60:
        #        return
        #log.info("Now regularizing %s with id %s for s1=%s and e1=%s"
        #         % (raw_series_db, raw_series_db.id, s1, e1))
        fifteen_min_series = regularize(raw_series_db, fifteen_min_series_db,
                                        s1, e1)
        if fifteen_min_series and fifteen_min_series.bounding_dates():
            bounds[variable]['fifteen_start'] = min(
                bounds[variable]['fifteen_start'],
                fifteen_min_series.bounding_dates()[0])
            bounds[variable]['fifteen_end'] = max(
                bounds[variable]['fifteen_end'],
                fifteen_min_series.bounding_dates()[1])

            result = fifteen_min_series
            monthly_series = None
            # Stefanos wants to aggregate using previous results
            # Why? The problem is that Monthly series is not inserted.
            # Was this on purpose?
            for time_step_id in (TSTEP_HOURLY, TSTEP_DAILY, TSTEP_MONTHLY):
                result = aggregate(household, result, time_step_id, variable)
                if time_step_id == TSTEP_MONTHLY:
                    monthly_series = result
            if not monthly_series:
                return
            # Cost calculation only if monthly_series present
            try:
                cost_timeseries_db = household \
                    .timeseries.get(time_step__id=TSTEP_MONTHLY,
                                    variable__id={
                                        VAR_PERIOD: VAR_COST,
                                        VAR_ENERGY_PERIOD: VAR_ENERGY_COST
                                    }[variable])
                cost_timeseries = calculate_cost(monthly_series,
                                                 rate={
                                                     VAR_PERIOD:
                                                     CUBIC_METER_FLAT_RATE,
                                                     VAR_ENERGY_PERIOD:
                                                     KWH_FLAT_RATE
                                                 }[variable])
                cost_timeseries.id = cost_timeseries_db.id
                cost_timeseries.write_to_db(db=db.connection, commit=True)
            except Exception as e:
                log.info("Error in monthly calculation %s. Skipping!" %
                         repr(e))
                continue
Esempio n. 10
0
def create_objects(data, usernames, force, z_names, z_dict):
    """

    :param data: meter_id -> consumption_type -> [timestamp, volume]
    :param force: True to overwrite
    :return: True for success
    """
    households = []
    # Create user (household owner), household, database series placeholders
    hh_ids = sorted(data.keys())
    found = False
    for hh_id in hh_ids:
        username = usernames[hh_id]
        if username == "PT94993":
            pass
        try:
            zone_name = z_dict[username]
        except KeyError:
            zone_name = z_names[0]
        zone = DMA.objects.get(name=zone_name)
        user, created = create_user(username, hh_id)
        household, found = create_household(hh_id, user, zone.id)
        households.append(household)
        db_series = create_raw_timeseries(household)
        create_processed_timeseries(household)
        timeseries_data = {}
        # Now we will create timeseries.Timeseries() and we will add
        # parsed values
        for variable in db_series:
            if variable not in ('WaterCold', 'Electricity'):
                continue
            exists = False
            s, e = timeseries_bounding_dates_from_db(db.connection,
                                                     db_series[variable].id)
            latest_ts = e
            ts_id = db_series[variable].id
            # checking to see if timeseries records already exist in order
            # to append
            # d = read_timeseries_tail_from_db(db.connection, ts_id)
            total = 0.0
            # if s or e:
            #     exists = True
            #     timeseries = TSeries(ts_id)
            #     timeseries.read_from_db(db.connection)
            # else:
            #     timeseries = TSeries()
            #     timeseries.id = ts_id
            _dict = data[hh_id]
            arr = _dict[variable]
            series = arr
            if not series:
                continue
            earlier = []
            if (not latest_ts) or (latest_ts < series[0][0]):  # append
                timeseries = TSeries()
                timeseries.id = ts_id
                try:
                    tail = read_timeseries_tail_from_db(db.connection, ts_id)
                    total = float(tail[1])  # keep up from last value
                except Exception as e:
                    log.debug(repr(e))
                    total = 0
                for timestamp, value in series:
                    if (not latest_ts) or (timestamp > latest_ts):
                        if not isnan(value):
                            total += value
                            timeseries[timestamp] = total
                        else:
                            timeseries[timestamp] = float('NaN')
                    elif timestamp < latest_ts:
                        earlier.append((timestamp, value))
                timeseries.append_to_db(db=db.connection,
                                        transaction=transaction,
                                        commit=True)
            elif latest_ts >= series[0][0]:
                if not force:  # ignore
                    continue
                else:  # insert
                    for timestamp, value in series:
                        if timestamp < latest_ts:
                            earlier.append((timestamp, value))
            if earlier and ("GR" in username
                            or "GBA" in username):  # insert (only for athens)
                # print "appending %s items for %s" % (len(earlier), username)
                if variable == "WaterCold":
                    ts15 = household \
                        .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                                        variable__id=VAR_PERIOD)
                    series15 = TSeries(id=ts15.id)
                elif variable == "Electricity":
                    ts15 = household \
                        .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                                        variable__id=VAR_ENERGY_PERIOD)
                    series15 = TSeries(id=ts15.id)
                series15.read_from_db(db.connection)
                for ts, value in earlier:
                    series15[ts] = value
                series15.write_to_db(db=db.connection,
                                     transaction=transaction,
                                     commit=True)

                raw_ts = TSeries(ts_id)  # read existing ts raw data
                raw_ts.read_from_db(db.connection)
                total = get_consumption_totals(household, earlier[0][0],
                                               variable)
                init = total
                for timestamp, value in earlier:
                    if not isnan(value):
                        total += value
                        raw_ts[timestamp] = total
                    else:
                        raw_ts[timestamp] = float('NaN')

                # correct later values, too
                diff = total - init
                all_ts = sorted(raw_ts.keys())
                for ts in all_ts:
                    if ts <= timestamp:
                        continue
                    curr = raw_ts[ts]
                    raw_ts[ts] = curr + diff

                raw_ts.write_to_db(db=db.connection,
                                   transaction=transaction,
                                   commit=True)

        if 'WaterCold' in timeseries_data and not found:  # only for new HH
            calc_occupancy(timeseries_data['WaterCold'], household)
    return households
Esempio n. 11
0
def process_household(household):
    for variable in (VAR_PERIOD, VAR_ENERGY_PERIOD):
        raw_series_db = household \
            .timeseries \
            .filter(time_step__isnull=True,
                    variable__id={
                        VAR_PERIOD: VAR_CUMULATIVE,
                        VAR_ENERGY_PERIOD: VAR_ENERGY_CUMULATIVE
                    }[variable])[:1]
        if not raw_series_db:
            continue
        raw_series_db = raw_series_db[0]
        fifteen_min_series_db = household \
            .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                            variable__id=variable)
        s1, e1 = timeseries_bounding_dates_from_db(db.connection,
                                                   raw_series_db.id)
        s2, e2 = timeseries_bounding_dates_from_db(db.connection,
                                                   fifteen_min_series_db.id)
        #    if e2 and (e1-e2).seconds<15*60:
        #        return
        #log.info("Now regularizing %s with id %s for s1=%s and e1=%s"
        #         % (raw_series_db, raw_series_db.id, s1, e1))
        fifteen_min_series = regularize(raw_series_db, fifteen_min_series_db,
                                        s1, e1)
        if fifteen_min_series and fifteen_min_series.bounding_dates():
            bounds[variable]['fifteen_start'] = min(
                bounds[variable]['fifteen_start'],
                fifteen_min_series.bounding_dates()[0])
            bounds[variable]['fifteen_end'] = max(
                bounds[variable]['fifteen_end'],
                fifteen_min_series.bounding_dates()[1])

            result = fifteen_min_series
            monthly_series = None
            # Stefanos wants to aggregate using previous results
            # Why? The problem is that Monthly series is not inserted.
            # Was this on purpose?
            for time_step_id in (TSTEP_HOURLY, TSTEP_DAILY, TSTEP_MONTHLY):
                result = aggregate(household, result, time_step_id, variable)
                if time_step_id == TSTEP_MONTHLY:
                    monthly_series = result
            if not monthly_series:
                return
            # Cost calculation only if monthly_series present
            try:
                cost_timeseries_db = household \
                    .timeseries.get(time_step__id=TSTEP_MONTHLY,
                                    variable__id={
                                        VAR_PERIOD: VAR_COST,
                                        VAR_ENERGY_PERIOD: VAR_ENERGY_COST
                                    }[variable])
                cost_timeseries = calculate_cost(monthly_series,
                                                 rate={
                                                     VAR_PERIOD: CUBIC_METER_FLAT_RATE,
                                                     VAR_ENERGY_PERIOD: KWH_FLAT_RATE
                                                 }[variable])
                cost_timeseries.id = cost_timeseries_db.id
                cost_timeseries.write_to_db(db=db.connection, commit=True)
            except Exception as e:
                log.info("Error in monthly calculation %s. Skipping!" % repr(e))
                continue
Esempio n. 12
0
def create_objects(data, usernames, force, z_names, z_dict):
    """

    :param data: meter_id -> consumption_type -> [timestamp, volume]
    :param force: True to overwrite
    :return: True for success
    """
    households = []
    # Create user (household owner), household, database series placeholders
    hh_ids = sorted(data.keys())
    found = False
    for hh_id in hh_ids:
        username = usernames[hh_id]
        if username == "PT94993":
            pass
        try:
            zone_name = z_dict[username]
        except KeyError:
            zone_name = z_names[0]
        zone = DMA.objects.get(name=zone_name)
        user, created = create_user(username, hh_id)
        household, found = create_household(hh_id, user, zone.id)
        households.append(household)
        db_series = create_raw_timeseries(household)
        create_processed_timeseries(household)
        timeseries_data = {}
        # Now we will create timeseries.Timeseries() and we will add
        # parsed values
        for variable in db_series:
            if variable not in ('WaterCold', 'Electricity'):
                continue
            exists = False
            s, e = timeseries_bounding_dates_from_db(db.connection,
                                                     db_series[variable].id)
            latest_ts = e
            ts_id = db_series[variable].id
            # checking to see if timeseries records already exist in order
            # to append
            # d = read_timeseries_tail_from_db(db.connection, ts_id)
            total = 0.0
            # if s or e:
            #     exists = True
            #     timeseries = TSeries(ts_id)
            #     timeseries.read_from_db(db.connection)
            # else:
            #     timeseries = TSeries()
            #     timeseries.id = ts_id
            _dict = data[hh_id]
            arr = _dict[variable]
            series = arr
            if not series:
                continue
            earlier = []
            if (not latest_ts) or (latest_ts < series[0][0]):  # append
                timeseries = TSeries()
                timeseries.id = ts_id
                try:
                    tail = read_timeseries_tail_from_db(db.connection, ts_id)
                    total = float(tail[1])  # keep up from last value
                except Exception as e:
                    log.debug(repr(e))
                    total = 0
                for timestamp, value in series:
                    if (not latest_ts) or (timestamp > latest_ts):
                        if not isnan(value):
                            total += value
                            timeseries[timestamp] = total
                        else:
                            timeseries[timestamp] = float('NaN')
                    elif timestamp < latest_ts:
                        earlier.append((timestamp, value))
                timeseries.append_to_db(db=db.connection,
                                        transaction=transaction,
                                        commit=True)
            elif latest_ts >= series[0][0]:
                if not force:  # ignore
                    continue
                else:  # insert
                    for timestamp, value in series:
                        if timestamp < latest_ts:
                            earlier.append((timestamp, value))
            if earlier and ("GR" in username or "GBA" in username):  # insert (only for athens)
                # print "appending %s items for %s" % (len(earlier), username)
                if variable == "WaterCold":
                    ts15 = household \
                        .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                                        variable__id=VAR_PERIOD)
                    series15 = TSeries(id=ts15.id)
                elif variable == "Electricity":
                    ts15 = household \
                        .timeseries.get(time_step__id=TSTEP_FIFTEEN_MINUTES,
                                        variable__id=VAR_ENERGY_PERIOD)
                    series15 = TSeries(id=ts15.id)
                series15.read_from_db(db.connection)
                for ts, value in earlier:
                    series15[ts] = value
                series15.write_to_db(db=db.connection,
                                     transaction=transaction,
                                     commit=True)

                raw_ts = TSeries(ts_id)  # read existing ts raw data
                raw_ts.read_from_db(db.connection)
                total = get_consumption_totals(household, earlier[0][0],
                                               variable)
                init = total
                for timestamp, value in earlier:
                    if not isnan(value):
                        total += value
                        raw_ts[timestamp] = total
                    else:
                        raw_ts[timestamp] = float('NaN')

                # correct later values, too
                diff = total - init
                all_ts = sorted(raw_ts.keys())
                for ts in all_ts:
                    if ts <= timestamp:
                        continue
                    curr = raw_ts[ts]
                    raw_ts[ts] = curr + diff

                raw_ts.write_to_db(db=db.connection,
                                   transaction=transaction,
                                   commit=True)

        if 'WaterCold' in timeseries_data and not found:  # only for new HH
            calc_occupancy(timeseries_data['WaterCold'], household)
    return households
Esempio n. 13
0
def process_household(household, bounds):
    """
    Processes each household time series, first by regularizing step
    of the raw series to the fifteen minutes time series, then by
    aggregating fifteen minutes series to hourly, daily, monthly.
    """
    print "Processing household: %s" % (household.alt_codes.all()[0], )
    for variable in (VAR_PERIOD, VAR_ENERGY_PERIOD):
        raw_series_db = household.timeseries.filter(time_step__isnull=True,
                                                    variable__id={
                                                        VAR_PERIOD:
                                                        VAR_CUMULATIVE,
                                                        VAR_ENERGY_PERIOD:
                                                        VAR_ENERGY_CUMULATIVE
                                                    }[variable])[:1]
        if not raw_series_db:
            continue
        raw_series_db = raw_series_db[0]
        fifteen_min_series_db = household.timeseries.get(
            time_step__id=TSTEP_FIFTEEN_MINUTES, variable__id=variable)
        s1, e1 = timeseries_bounding_dates_from_db(db.connection,
                                                   raw_series_db.id)
        s2, e2 = timeseries_bounding_dates_from_db(db.connection,
                                                   fifteen_min_series_db.id)
        #    if e2 and (e1-e2).seconds<15*60:
        #        return
        fifteen_min_series = regularize_raw_series(raw_series_db,
                                                   fifteen_min_series_db, s1,
                                                   e1, s2, e2)
        if fifteen_min_series.bounding_dates():
            bounds[variable]['fifteen_start'] = min(
                bounds[variable]['fifteen_start'],
                fifteen_min_series.bounding_dates()[0])
            bounds[variable]['fifteen_end'] = max(
                bounds[variable]['fifteen_end'],
                fifteen_min_series.bounding_dates()[1])
        result = fifteen_min_series
        monthly_series = None
        for time_step_id in (TSTEP_HOURLY, TSTEP_DAILY, TSTEP_MONTHLY):
            if not result:
                break
            result = aggregate_household_series(bounds, household, result,
                                                time_step_id, variable)
            if time_step_id == TSTEP_MONTHLY:
                monthly_series = result
        if not monthly_series:
            return
        # Cost calculation only if monthly_series present
        cost_timeseries_db = household.timeseries.get(
            time_step__id=TSTEP_MONTHLY,
            variable__id={
                VAR_PERIOD: VAR_COST,
                VAR_ENERGY_PERIOD: VAR_ENERGY_COST
            }[variable])
        cost_timeseries = calculate_cost_timeseries(monthly_series,
                                                    rate={
                                                        VAR_PERIOD:
                                                        CUBIC_METER_FLAT_RATE,
                                                        VAR_ENERGY_PERIOD:
                                                        KWH_FLAT_RATE
                                                    }[variable])
        cost_timeseries.id = cost_timeseries_db.id
        cost_timeseries.write_to_db(db=db.connection, commit=True)
Esempio n. 14
0
def usage_data(request, ctr_code):
    user = request.user
    if user.is_superuser:
        users = User.objects.filter(username__startswith=ctr_code)
        u_data = {}
        user_pages = {}
        for user in users:
            arr = [user.username]
            try:
                household = Household.objects.get(user=user)
            except Household.DoesNotExist:
                continue
            for variable in (VAR_PERIOD, VAR_ENERGY_PERIOD):
                if variable == VAR_PERIOD:
                    _type = "WATER"
                else:
                    _type = "ENERGY"
                raw_series_db = household.timeseries.filter(
                    time_step__isnull=True,
                    variable__id={
                        VAR_PERIOD: VAR_CUMULATIVE,
                        VAR_ENERGY_PERIOD: VAR_ENERGY_CUMULATIVE
                    }[variable])[:1]
                if not raw_series_db:
                    continue
                raw_series_db = raw_series_db[0]
                s1, e1 = timeseries_bounding_dates_from_db(
                    db.connection, raw_series_db.id)
                if s1:
                    start = s1.strftime("%Y-%m-%d %H:%M")
                else:
                    start = ""
                if e1:
                    end = e1.strftime("%Y-%m-%d %H:%M")
                else:
                    end = ""
                arr.extend([start, end])
            ud = UsageData.objects.filter(user=user)
            if ud:
                ets = ud[0].enter_ts
                lts = ud[0].exit_ts
                arr.append(ets.strftime("%Y-%m-%d %H:%M"))
                arr.append(lts.strftime("%Y-%m-%d %H:%M"))
                # duration
                arr.append(int(time.mktime(lts.timetuple())
                               - time.mktime(ets.timetuple())) / 60)
            else:
                arr.extend([0, 0, 0])
            upvs = UserPageView.objects.filter(user=user).order_by('-added')
            if upvs:
                arr.append(upvs[0].added)
            else:
                arr.append("-")
            arr.append(len(ud))
            u_data[user.id] = arr
            pages = {}
            for upv in upvs:
                page = upv.page
                count = pages.get(page, 0)
                count += 1
                pages[page] = count
            keys = pages.keys()
            page_tuples = []
            for k in keys:
                page_tuples.append((k, pages[k]))
            user_pages[user.id] = page_tuples
        data = {
            "users": users,
            "data": u_data,
            "pages": user_pages,
        }
        variables = RequestContext(request, data)
        return render_to_response("_usage_data.html", variables)
    else:
        return HttpResponseRedirect(reverse('dashboard'))