Example #1
0
        def _get_candles(pair, candle_type, start_date, end_date):
            '''
            pubhubのget_candlestickを複数日扱えるように拡張
            start 09:00 - end 8:55
            '''

            start_date = datetime.strptime(start_date, '%Y-%m-%d')
            end_date = datetime.strptime(end_date, '%Y-%m-%d')

            candles = []
            for d in daterange(start_date, end_date + timedelta(1)):
                tmp = pub.get_candlestick(
                    pair, candle_type,
                    d.strftime('%Y%m%d'))['candlestick'][0]['ohlcv']
                if d == start_date:
                    candles = tmp
                else:
                    candles = merge_list(candles, tmp)
            candles = np.array(candles).astype(np.float64)
            return candles
def generate_water_register(crop_season, 
                            field, 
                            user, 
                            start_date=None, 
                            report_date=None,
                            force=False):

    ####
    ## Determine planting date, and stop calculation if no planting has been done
    planting_event = CropSeasonEvent.objects.filter(crop_season=crop_season,
                                                    field=field,
                                                    crop_event__name='Planting').order_by("-date").first()
    if not planting_event: return 0
    ##
    ####

    ####
    ## Determine the first and last first event date to compute
    if start_date is None:
        start_date = planting_event.date

    if report_date is None:
        report_date = datetime.now().date()

    if COMPUTE_FULL_SEASON:
        end_date = crop_season.season_end_date + timedelta(1)
    else:
        today_plus_delta = report_date + timedelta(days=WATER_REGISTER_DELTA)
        latest_water_register = WaterRegister.objects.filter(crop_season=crop_season,
                                                             field=field
                                                             ).order_by('datetime').last()

        if latest_water_register:
            last_register_date = max(today_plus_delta, latest_water_register.datetime.date())
        else:
            last_register_date = today_plus_delta

        end_date = min(last_register_date, crop_season.season_end_date) + timedelta(1)
    ##
    ####
    ## Cache values / queries for later use

    water_history_query       = WaterHistory.objects.filter(crop_season=crop_season,
                                                            field=field,
                                                            ignore = False)[:]
    crop_season_events_query = CropSeasonEvent.objects.filter(crop_season=crop_season, 
                                                              field=field).distinct().select_related('crop_event')[:]
    ####

    

    if force:
        ## force recomputation from first date
        first_process_date = start_date
    else:
        ## Use modification times to find the earliest water register to update
        first_process_date = max(crop_season.season_start_date,
                                 earliest_register_to_update(report_date, crop_season, field))

    wr_query = WaterRegister.objects.filter(crop_season=crop_season,
                                            field=field, 
                                            datetime__gte=d2dt_min(first_process_date),
                                            datetime__lte=d2dt_max(end_date)
                                            ).all()

    maxWater = float(field.soil_type.max_available_water)
    minWater = calculateAWC_min( crop_season, field )

    if first_process_date <= crop_season.season_start_date:
        ## Assume that each field starts with a full water profile
        AWC_initial = maxWater
    else:
        try:
            wr_yesterday = WaterRegister.objects.get(crop_season=crop_season,
                                                     field=field,
                                                     datetime__range=d2dt_range(first_process_date - timedelta(days=1))
                                                     )
            AWC_initial = wr_yesterday.average_water_content
        except:
            raise RuntimeError("No previous water_register record on " + str(first_process_date) );

    ## First pass, calculate water profile (AWC)
    #if DEBUG: print "First pass, calculate water profile (AWC)"
    temps_since_last_water_date = []
    wr_prev = None

    # if DEBUG: print "Date range: %s to %s" % (first_process_date, end_date)
    ## Some optimization to do here: After the first pass we know the prev record is there.
    for  date in daterange(first_process_date, end_date):
        ####
        ## Get AWC for yesterday, and copy the irrigate_to_max_seen, irrigate_to_max_achieved flags
        ##
        yesterday = date - timedelta(days=1)

        ## Check if we have (cached) the water register object for
        ## yesterday, if so grab the AWC, otherwise use the default
        ## maximum for the soil type
        if (wr_prev is None) or (wr_prev.date != yesterday): 
            try:
                wr_prev = wr_query.filter(datetime__range=d2dt_range(yesterday))[0]
                AWC_prev = wr_prev.average_water_content
                irrigate_to_max_seen_prev = wr_prev.irrigate_to_max_seen
                irrigate_to_max_achieved_prev = wr_prev.irrigate_to_max_achieved
            except ( ObjectDoesNotExist,  IndexError, ):
                AWC_prev = AWC_initial
                irrigate_to_max_seen_prev = False
                irrigate_to_max_achieved_prev = False

        else:
            AWC_prev = wr_prev.average_water_content
            irrigate_to_max_seen_prev = wr_prev.irrigate_to_max_seen
            irrigate_to_max_achieved_prev = wr_prev.irrigate_to_max_achieved

            irrigate_to_max_seen_prev = wr_prev.irrigate_to_max_seen
            irrigate_to_max_achieved_prev = wr_prev.irrigate_to_max_achieved

        ####
        

        ####
        ## Get or Create a water register object (db record) for today
        
        ## Delete previous water register entries
        # wr_query.filter(datetime__range=d2dt_range(date)).delete()
        # wr = WaterRegister(crop_season = crop_season,
        #                    field = field,
        #                    datetime__range = d2dt_range(date)
        #                    )

        try: 
            wr = wr_query.filter(datetime__range=d2dt_range(date))[0]
            computed_from_probes  = False
            irrigate_flag         = False
            too_hot_flag          = False
            check_sensors_flag    = False
            dry_down_flag         = False

            # clear out existing flags
            wr.irrigate_flag = False
            wr.too_hot_flag  = False
            wr.days_to_irrigation = -1
            wr.check_sensors_flag = False
            
        except ( ObjectDoesNotExist,  IndexError, ):
            wr = WaterRegister(
                crop_season = crop_season,
                field = field,
                datetime = d2dt_min(date)
            )

        ####
        ## Copy information from crop event record 
        ## 
        ## * It is currently possible to assign out-of-order
        ## * CropSeasonEvent dates.  To manage this, first select
        ## * CropSeasonEvents that occur _on_or_before_ today, then select the 
        ## * last by CropEvent.order 
        cse = crop_season_events_query.filter(date__lte=date).order_by('crop_event__order').last()
        ## if DEBUG: print cse
	if cse is None:
            print "No CropSeasonEvent defined for %s %s on %s.  Deleting WaterRegister record and skipping computation." % ( crop_season, field, date )
            del(wr)
            continue

        ce = cse.crop_event

        wr.crop_stage      = ce.name
        wr.daily_water_use = ce.daily_water_use
        wr.max_temp_2in    = ce.max_temp_2in
        wr.irrigate_to_max = ce.irrigate_to_max
        wr.do_not_irrigate = ce.do_not_irrigate
        wr.message         = ce.irrigation_message
        # if DEBUG: 
        #     print "Date: %s  ce.Crop Stage: %s" % ( date, ce.name )
        #     print "Date: %s  wr.Crop Stage: %s" % ( date, wr.crop_stage )

        ##
        ####

        ####
        ## Get (automatic) probe reading information and calculate AWC

        #AWC_probe = None
        #temp = None

        # Moved this validation to calculateAWC,
        # since there is already code to validate.
        AWC_probe = calculateAWC(crop_season,
                                 field,
                                 date,
                                 water_history_query)

        # if DEBUG: print "  AWC_probe=", AWC_probe
        ##
        ####

        ####
        ## Get (manually entered) water register entries
        wr.rain, wr.irrigation, wr.min_temp_24_hours, wr.max_temp_24_hours  = calculate_total_RainIrrigation(crop_season,
                                                                                                             field,
                                                                                                             date, 
                                                                                                             water_history_query)
        
        AWC_register = float(AWC_prev) - float(wr.daily_water_use) + float(wr.rain) + float(wr.irrigation)
        ##if DEBUG: print "  AWC_register=", AWC_register
        ##
        ####

        ####
        ## Prefer AWC from probe reading over AWC from water registry
        if AWC_probe is not None: 
            wr.average_water_content = quantize(AWC_probe)
            wr.computed_from_probes  = True
        else:
            wr.average_water_content = quantize(AWC_register)
            wr.computed_from_probes  = False
        # if DEBUG: print "  wr.average_water_content=", wr.average_water_content
        # if DEBUG: print "  wr.computed_from_probes=", wr.computed_from_probes
        ##
        ####

        ## Enforce min and maximum soil water content based on soil type
        if wr.average_water_content > maxWater: 
            #if DEBUG: print "  Enforce max soil AWC: ", maxWater
            wr.average_water_content = maxWater

        if wr.average_water_content < minWater: 
            #if DEBUG: print "  Enforce min soil AWC: ", minWater
            wr.average_water_content = minWater

        ## Store user into accounting info..
        if wr.cuser_id is None:
            wr.cuser_id = user.pk
        wr.muser_id = user.pk


        ## Calculate and store max temperature since last appreciable rainfall or irrigation
        if wr.average_water_content >= float(AWC_prev) + 0.1:
            # Max temp is only today's value 
            wr.max_observed_temp_2in = wr.max_temp_24_hours

            # Reset max temp calculation
            temps_since_last_water_date = []
        else:
            # Add today's temperature
            temps_since_last_water_date.append(wr.max_temp_24_hours)

            # Calculate max temp
            wr.max_observed_temp_2in = max(temps_since_last_water_date)

        ## Cache this entry for tomorrow
        wr_prev = wr

        ## Write to the database
        wr.save()

    ## Refresh query
    wr_query = WaterRegister.objects.filter(crop_season=crop_season,
                                            field=field, 
                                            datetime__gte=d2dt_min(first_process_date),
                                            datetime__lte=d2dt_max(end_date)
                                            ).all()
        
    ## Second pass, calculate flags 
    #if DEBUG: print "Second pass, calculate flags"
    irrigate_to_max_flag_seen = False
    irrigate_to_max_achieved  = False
    drydown_flag              = False
    irrigate_to_max_days      = 0
    nChanged                  = 0
    for date in daterange(first_process_date, end_date):

        if not wr_query.filter(datetime__range=d2dt_range(date)): continue
        wr = wr_query.filter(datetime__range=d2dt_range(date))[0]

        ## Will handle both the case where the first irrigate_to_flag set to 
        ## true was in a register not updated for this report.
        if wr.irrigate_to_max or wr.irrigate_to_max_seen: 
            irrigate_to_max_flag_seen = True
            wr.irrigate_to_max_seen = True

        #####
        ## If the irrigate_to_max flag has been seen, irrigate & check
        ## sensors until maxWater is achieved (or 3 watering days have
        ## occured), then no more irrigation and no more sensor checks   
        ## ('drydown') 
        #####
        if not irrigate_to_max_flag_seen:
            ####
            ## Check if we need to irrigate *today*

            # clear out existing flags
            wr.irrigate_flag = False
            wr.too_hot_flag  = False
            wr.days_to_irrigation = -1
            wr.check_sensors_flag = False

            # never irrigate if flag is set
            if wr.do_not_irrigate:
                wr.irrigate_flag = False
            else:
                # Too dry:
                if wr.average_water_content <= 0.00:
                    wr.irrigate_flag = True
                    wr.days_to_irrigation = 0
            
                # Too hot:
                if wr.max_temp_2in is not None and \
                   wr.max_observed_temp_2in is not None and \
                   wr.max_observed_temp_2in > wr.max_temp_2in:
                    wr.irrigate_flag = True
                    wr.days_to_irrigation = 0
                    wr.too_hot_flag  = True

                ####
                ## Check if we need to irrigate in the next few days, based on WATER_REGISTER_DELTA
                ####
                wr.check_sensors_flag = False
                if wr.days_to_irrigation < 0:
                    date_plus_delta = date + timedelta(days=(WATER_REGISTER_DELTA+1))
                    for date_future in daterange(date + timedelta(days=1), date_plus_delta):
                        try:
                            wr_future = wr_query.get(datetime__range=d2dt_range(date_future))
                            #wr.check_sensors_flag = wr.check_sensors_flag or (wr_future.average_water_content <= 0.00)
                            if ( wr_future.average_water_content <= 0.00 ) and ( wr_future.do_not_irrigate == False ):
                                wr.check_sensors_flag = True
                                wr.days_to_irrigation = (date_future - date).days
                                break
                        except ObjectDoesNotExist:
                            pass

        else:  # execute below if irrigate_to_max_flag_seen is true
            if wr.irrigate_to_max_achieved or irrigate_to_max_achieved or irrigate_to_max_days >= 3:
                wr.irrigate_flag = False
                wr.dry_down_flag = True
            else:
                irrigate_to_max_days += 1
                if wr.average_water_content >= maxWater:
                    wr.irrigate_to_max_achieved = True
                    irrigate_to_max_achieved = True
                    wr.irrigate_flag         = False
                    wr.check_sensors_flag    = False
                    wr.dry_down_flag         = True
                else:
                    wr.irrigate_flag      = True
                    wr.check_sensors_flag = True

        ## Write to the database
        wr.save()
        nChanged += 1


    # reset the dependency date
    field.earliest_changed_dependency_date = None
    field.save()

    return nChanged
def generate_objects(wh_formset, crop_season, field, user,  report_date):

    """
    Returns the list of object for display in the template.
    Each object is an instance of the class UnifiedReport,
    defined below.

    We assume that the wh_formset is sorted by date, but not necessarily
    by source. In this implementation there are two possible sources for 
    the WaterHistory objects for one date: at most one UGA and any number
    of User.
    """

    if crop_season.season_start_date > report_date:
        return None
    
    if crop_season.season_end_date < report_date:
        report_date = crop_season.season_end_date

    generate_water_register(crop_season, field, user, None, report_date)

    water_register_query = WaterRegister.objects.filter(crop_season = crop_season, field = field)
    water_register_query = water_register_query.order_by('-datetime')
    water_register_query = water_register_query.filter(
                               Q(datetime__lte =  d2dt_min(report_date + timedelta(WATER_REGISTER_DELTA))) 
                               )

    if len(water_register_query) == 0:
        return None

    ### Get all the forms defined by the formset created from the water history objects
    form_index = 0
    current_form = None
    all_forms = wh_formset.forms
    ret = []

    if all_forms is not None and len(all_forms) > 0:
        current_form = all_forms[form_index]
        form_index = form_index + 1

        ## Point to the first form in the current crop season
        while current_form is not None and \
              getDateObject(current_form['datetime'].value()) is not None and \
              getDateObject(current_form['datetime'].value()) < crop_season.season_start_date:
            if form_index == len(all_forms):
                current_form = None
            else:
                print "IF WE SEE THIS THEN THIS LOOP IS NECESSARY"
                current_form = all_forms[form_index]
                form_index = form_index + 1
                

    for day in daterange(crop_season.season_start_date, report_date + timedelta(days=1)):
        try:
            water_register = water_register_query.get(datetime__range = d2dt_range(day))
        except:
            print "Some date in crop season does not have a water register. Return nothing."
            return None

        day_record = UnifiedReport(day, water_register)

        while current_form is not None and getDateObject(current_form['datetime'].value()) == day:
            if current_form['source'].value() == "UGA":
                if day_record.uga_form is not None:
                    message = "There is more than one UGA probe defined for %s on %s " % \
                              (crop_season, day)
                    #raise RuntimeError(message)
                    print message
                day_record.add_uga(current_form)

            elif current_form['source'].value() == "User":
                day_record.all_forms.append(current_form)
            else:
                # raise RuntimeError("Unrecogized source type: " + current_form['source'].value())
                print "Have a WaterHistory of 'Unknown' type: ", current_form
                day_record.all_forms.append(current_form)

            if form_index == len(all_forms):
                current_form = None
            else:
                current_form = all_forms[form_index]
                form_index = form_index + 1

        ret.append(day_record)

    # Add records for days in the future
    
    ### Might want days=WATER_REGISTER_DELTA+1 below, but we don't do it
    ### in generate_water_register

    ### Also this loop could be merged with above. But this is easier to see
    ### what happens
    report_plus_delta = min(report_date + timedelta(days=WATER_REGISTER_DELTA), crop_season.season_end_date)
    
    for day in daterange(report_date + timedelta(days=1), report_plus_delta):
        water_register = water_register_query.filter(datetime__range = d2dt_range(day))
        if len(water_register) == 1:
            day_record = UnifiedReport(day, water_register[0])
            ret.append(day_record)
        else:
            print "Found %d WaterRegister entries on %s for %s " % ( len(water_register),
                                                                     day,
                                                                     crop_season )
    return  ret
Example #4
0
    def courses(self, request, start, end):
        """
        /courses/:start/:end

        QUERY PARAMS
        ------------
        include: ','-separated list of 'deleted', 'added', 'rescheduled', 'offdays'
            Choose what types of mutation-affected course to show.
            Unaffected courses are always included.
            Courses are marked as such with the `mutation` attribute
            Default: added,rescheduled,deleted

        week-type: ','-separated list of 'Q1', 'Q2' | 'auto'
            Choose which week types to include.
            'auto': only shows the current week type.
            Default: Q1
        
        TODO(beta-1.1.0): handle /courses/:start
        TODO(beta-1.1.0): handle ?week-type=auto
        MAYBE(beta-1.1.0): use ?start & ?end instead of url fragments
        """
        user = request.user

        #
        # Query params processing
        #

        # Get the date format and parse accordingly
        def parse_iso8601(value):
            fmt = 'datetime' if 'T' in value else 'date'
            if '/' in value:
                value = datetime.datetime.strptime(value, '%d/%m/%Y')
            else:
                value = datetime.datetime.fromisoformat(value)
            return (fmt, value)

        # Processing start & end
        start_fmt, start = parse_iso8601(start)
        end_fmt, end = parse_iso8601(end)
        if start_fmt != end_fmt:
            return Response({
                'error':
                'Date range bounds must have the same format',
            })

        using_time = start_fmt == 'datetime'  # We could have picked end_fmt since start_fmt == end_fmt

        # Processing include
        include = request.query_params.get('include',
                                           'added,rescheduled,deleted')
        include = [i.strip() for i in include.split(',')]
        # Translating query parameter's mutation types into internal ones used by .models.Mutation
        mutation_types_map = {
            'deleted': 'DEL',
            'rescheduled': 'RES',
            'added': 'ADD'
        }
        include = [mutation_types_map.get(i, i) for i in include]

        # Processing week-type
        week_types = request.query_params.get('week-type', 'Q1')
        week_types = [w.strip() for w in week_types.split(',')]

        #
        # Additionnal data processing
        #

        # Getting offdays as date ranges
        # Get the raw setting
        try:
            offdays = Setting.objects.get(setting__key='offdays').value
        except Setting.DoesNotExist:
            offdays = []
        else:
            # Each line is a new daterange/date
            offdays = offdays.split('\n')
            # Split to get start & end
            offdays = [o.split(' - ') for o in offdays]
            # For simple dates, set start & end to the same date
            offdays = [[d[0], d[0]] for d in offdays if len(d) == 1]
            # Parse the offdays' start & end dates into datetime.date objects
            offdays = [[parse_iso8601(d.strip())[1] for d in o]
                       for o in offdays]

        #
        # Collecting events
        #

        # Init the variable containing all the courses
        courses = []

        # Loop over each day of the range
        for day in daterange(start, end):
            # TODO: handle ADD mutations
            # Get the relevant events
            events = Event.objects.filter(
                Q(week_type__in=week_types) | Q(week_type='BOTH'),
                Q(day=int(day.strftime('%u'))), Q(subject__user__id=user.id))
            # Set the date part of each event, instead of a HH:MM time. (for start & end)
            for event in events:
                # Combine the date parts and the time parts
                event.start = datetime.datetime.combine(
                    day.date(),  # The loop's day (date part)
                    event.start  # The event's start time (time part)
                )
                # Do the same for the end
                event.end = datetime.datetime.combine(day.date(), event.end)

                # Check if the event is in offdays
                event.is_offday = False
                # For each offday daterange
                for offday in offdays:
                    offday_start, offday_end = offday
                    # Check if the event is in it
                    # If its already an offday because of another offday daterange, no need to check.
                    if not event.is_offday:
                        event.is_offday = dateranges_overlap(
                            (event.start, event.end),
                            (offday_start, offday_end))

                # Don't add the course to the list if the conditions aren't met
                if event.is_offday and 'offdays' not in include:
                    continue

                # Check if the event matches a mutation
                # (for EDT/RES/DEL mutations)
                # for ADD mutations, the mutation isn't bounded to an event.
                # We treat this separetly, and add them to the courses list outside of the events loop
                event.mutation = None
                for mutation in event.mutations.all():
                    # Determine if the mutation's date ranges matches the event
                    deleted_matches = False
                    added_matches = False
                    # Mutation types: see models.Mutation
                    if mutation.type in ('DEL', 'RES'):
                        deleted_matches = dateranges_overlap(
                            (mutation.deleted_start, mutation.deleted_end),
                            (event.start, event.end))
                    elif mutation.type == 'RES':
                        added_matches = dateranges_overlap(
                            (mutation.added_start, mutation.added_end),
                            (event.start, event.end))
                    # There should be at most one mutation relating to a single _course_ (not event).
                    # We set the mutation without further checking.
                    event.mutation = mutation

                event.original_room = event.room
                if event.mutation is not None:
                    # Process EDT/RES mutations that could modify the room
                    if event.mutation.type in (
                            'EDT', 'RES') and event.mutation.room is not None:
                        event.room = event.mutation.room

                    # Don't append courses whose mutation's type is not in include
                    if event.mutation.type not in include:
                        continue
                # Append the course to the list
                courses.append(event)

        # Return the response
        return Response(
            [CourseReadSerializer(course).data for course in courses])