def _get_candles(pair, candle_type, start_date, end_date):
'''
pubhubのget_candlestickを複数日扱えるように拡張
start 09:00 - end 8:55
'''
start_date = datetime.strptime(start_date, '%Y-%m-%d')
end_date = datetime.strptime(end_date, '%Y-%m-%d')
candles = []
for d in daterange(start_date, end_date + timedelta(1)):
tmp = pub.get_candlestick(
pair, candle_type,
d.strftime('%Y%m%d'))['candlestick'][0]['ohlcv']
if d == start_date:
candles = tmp
else:
candles = merge_list(candles, tmp)
candles = np.array(candles).astype(np.float64)
return candles
def generate_water_register(crop_season,
field,
user,
start_date=None,
report_date=None,
force=False):
####
## Determine planting date, and stop calculation if no planting has been done
planting_event = CropSeasonEvent.objects.filter(crop_season=crop_season,
field=field,
crop_event__name='Planting').order_by("-date").first()
if not planting_event: return 0
##
####
####
## Determine the first and last first event date to compute
if start_date is None:
start_date = planting_event.date
if report_date is None:
report_date = datetime.now().date()
if COMPUTE_FULL_SEASON:
end_date = crop_season.season_end_date + timedelta(1)
else:
today_plus_delta = report_date + timedelta(days=WATER_REGISTER_DELTA)
latest_water_register = WaterRegister.objects.filter(crop_season=crop_season,
field=field
).order_by('datetime').last()
if latest_water_register:
last_register_date = max(today_plus_delta, latest_water_register.datetime.date())
else:
last_register_date = today_plus_delta
end_date = min(last_register_date, crop_season.season_end_date) + timedelta(1)
##
####
## Cache values / queries for later use
water_history_query = WaterHistory.objects.filter(crop_season=crop_season,
field=field,
ignore = False)[:]
crop_season_events_query = CropSeasonEvent.objects.filter(crop_season=crop_season,
field=field).distinct().select_related('crop_event')[:]
####
if force:
## force recomputation from first date
first_process_date = start_date
else:
## Use modification times to find the earliest water register to update
first_process_date = max(crop_season.season_start_date,
earliest_register_to_update(report_date, crop_season, field))
wr_query = WaterRegister.objects.filter(crop_season=crop_season,
field=field,
datetime__gte=d2dt_min(first_process_date),
datetime__lte=d2dt_max(end_date)
).all()
maxWater = float(field.soil_type.max_available_water)
minWater = calculateAWC_min( crop_season, field )
if first_process_date <= crop_season.season_start_date:
## Assume that each field starts with a full water profile
AWC_initial = maxWater
else:
try:
wr_yesterday = WaterRegister.objects.get(crop_season=crop_season,
field=field,
datetime__range=d2dt_range(first_process_date - timedelta(days=1))
)
AWC_initial = wr_yesterday.average_water_content
except:
raise RuntimeError("No previous water_register record on " + str(first_process_date) );
## First pass, calculate water profile (AWC)
#if DEBUG: print "First pass, calculate water profile (AWC)"
temps_since_last_water_date = []
wr_prev = None
# if DEBUG: print "Date range: %s to %s" % (first_process_date, end_date)
## Some optimization to do here: After the first pass we know the prev record is there.
for date in daterange(first_process_date, end_date):
####
## Get AWC for yesterday, and copy the irrigate_to_max_seen, irrigate_to_max_achieved flags
##
yesterday = date - timedelta(days=1)
## Check if we have (cached) the water register object for
## yesterday, if so grab the AWC, otherwise use the default
## maximum for the soil type
if (wr_prev is None) or (wr_prev.date != yesterday):
try:
wr_prev = wr_query.filter(datetime__range=d2dt_range(yesterday))[0]
AWC_prev = wr_prev.average_water_content
irrigate_to_max_seen_prev = wr_prev.irrigate_to_max_seen
irrigate_to_max_achieved_prev = wr_prev.irrigate_to_max_achieved
except ( ObjectDoesNotExist, IndexError, ):
AWC_prev = AWC_initial
irrigate_to_max_seen_prev = False
irrigate_to_max_achieved_prev = False
else:
AWC_prev = wr_prev.average_water_content
irrigate_to_max_seen_prev = wr_prev.irrigate_to_max_seen
irrigate_to_max_achieved_prev = wr_prev.irrigate_to_max_achieved
irrigate_to_max_seen_prev = wr_prev.irrigate_to_max_seen
irrigate_to_max_achieved_prev = wr_prev.irrigate_to_max_achieved
####
####
## Get or Create a water register object (db record) for today
## Delete previous water register entries
# wr_query.filter(datetime__range=d2dt_range(date)).delete()
# wr = WaterRegister(crop_season = crop_season,
# field = field,
# datetime__range = d2dt_range(date)
# )
try:
wr = wr_query.filter(datetime__range=d2dt_range(date))[0]
computed_from_probes = False
irrigate_flag = False
too_hot_flag = False
check_sensors_flag = False
dry_down_flag = False
# clear out existing flags
wr.irrigate_flag = False
wr.too_hot_flag = False
wr.days_to_irrigation = -1
wr.check_sensors_flag = False
except ( ObjectDoesNotExist, IndexError, ):
wr = WaterRegister(
crop_season = crop_season,
field = field,
datetime = d2dt_min(date)
)
####
## Copy information from crop event record
##
## * It is currently possible to assign out-of-order
## * CropSeasonEvent dates. To manage this, first select
## * CropSeasonEvents that occur _on_or_before_ today, then select the
## * last by CropEvent.order
cse = crop_season_events_query.filter(date__lte=date).order_by('crop_event__order').last()
## if DEBUG: print cse
if cse is None:
print "No CropSeasonEvent defined for %s %s on %s. Deleting WaterRegister record and skipping computation." % ( crop_season, field, date )
del(wr)
continue
ce = cse.crop_event
wr.crop_stage = ce.name
wr.daily_water_use = ce.daily_water_use
wr.max_temp_2in = ce.max_temp_2in
wr.irrigate_to_max = ce.irrigate_to_max
wr.do_not_irrigate = ce.do_not_irrigate
wr.message = ce.irrigation_message
# if DEBUG:
# print "Date: %s ce.Crop Stage: %s" % ( date, ce.name )
# print "Date: %s wr.Crop Stage: %s" % ( date, wr.crop_stage )
##
####
####
## Get (automatic) probe reading information and calculate AWC
#AWC_probe = None
#temp = None
# Moved this validation to calculateAWC,
# since there is already code to validate.
AWC_probe = calculateAWC(crop_season,
field,
date,
water_history_query)
# if DEBUG: print " AWC_probe=", AWC_probe
##
####
####
## Get (manually entered) water register entries
wr.rain, wr.irrigation, wr.min_temp_24_hours, wr.max_temp_24_hours = calculate_total_RainIrrigation(crop_season,
field,
date,
water_history_query)
AWC_register = float(AWC_prev) - float(wr.daily_water_use) + float(wr.rain) + float(wr.irrigation)
##if DEBUG: print " AWC_register=", AWC_register
##
####
####
## Prefer AWC from probe reading over AWC from water registry
if AWC_probe is not None:
wr.average_water_content = quantize(AWC_probe)
wr.computed_from_probes = True
else:
wr.average_water_content = quantize(AWC_register)
wr.computed_from_probes = False
# if DEBUG: print " wr.average_water_content=", wr.average_water_content
# if DEBUG: print " wr.computed_from_probes=", wr.computed_from_probes
##
####
## Enforce min and maximum soil water content based on soil type
if wr.average_water_content > maxWater:
#if DEBUG: print " Enforce max soil AWC: ", maxWater
wr.average_water_content = maxWater
if wr.average_water_content < minWater:
#if DEBUG: print " Enforce min soil AWC: ", minWater
wr.average_water_content = minWater
## Store user into accounting info..
if wr.cuser_id is None:
wr.cuser_id = user.pk
wr.muser_id = user.pk
## Calculate and store max temperature since last appreciable rainfall or irrigation
if wr.average_water_content >= float(AWC_prev) + 0.1:
# Max temp is only today's value
wr.max_observed_temp_2in = wr.max_temp_24_hours
# Reset max temp calculation
temps_since_last_water_date = []
else:
# Add today's temperature
temps_since_last_water_date.append(wr.max_temp_24_hours)
# Calculate max temp
wr.max_observed_temp_2in = max(temps_since_last_water_date)
## Cache this entry for tomorrow
wr_prev = wr
## Write to the database
wr.save()
## Refresh query
wr_query = WaterRegister.objects.filter(crop_season=crop_season,
field=field,
datetime__gte=d2dt_min(first_process_date),
datetime__lte=d2dt_max(end_date)
).all()
## Second pass, calculate flags
#if DEBUG: print "Second pass, calculate flags"
irrigate_to_max_flag_seen = False
irrigate_to_max_achieved = False
drydown_flag = False
irrigate_to_max_days = 0
nChanged = 0
for date in daterange(first_process_date, end_date):
if not wr_query.filter(datetime__range=d2dt_range(date)): continue
wr = wr_query.filter(datetime__range=d2dt_range(date))[0]
## Will handle both the case where the first irrigate_to_flag set to
## true was in a register not updated for this report.
if wr.irrigate_to_max or wr.irrigate_to_max_seen:
irrigate_to_max_flag_seen = True
wr.irrigate_to_max_seen = True
#####
## If the irrigate_to_max flag has been seen, irrigate & check
## sensors until maxWater is achieved (or 3 watering days have
## occured), then no more irrigation and no more sensor checks
## ('drydown')
#####
if not irrigate_to_max_flag_seen:
####
## Check if we need to irrigate *today*
# clear out existing flags
wr.irrigate_flag = False
wr.too_hot_flag = False
wr.days_to_irrigation = -1
wr.check_sensors_flag = False
# never irrigate if flag is set
if wr.do_not_irrigate:
wr.irrigate_flag = False
else:
# Too dry:
if wr.average_water_content <= 0.00:
wr.irrigate_flag = True
wr.days_to_irrigation = 0
# Too hot:
if wr.max_temp_2in is not None and \
wr.max_observed_temp_2in is not None and \
wr.max_observed_temp_2in > wr.max_temp_2in:
wr.irrigate_flag = True
wr.days_to_irrigation = 0
wr.too_hot_flag = True
####
## Check if we need to irrigate in the next few days, based on WATER_REGISTER_DELTA
####
wr.check_sensors_flag = False
if wr.days_to_irrigation < 0:
date_plus_delta = date + timedelta(days=(WATER_REGISTER_DELTA+1))
for date_future in daterange(date + timedelta(days=1), date_plus_delta):
try:
wr_future = wr_query.get(datetime__range=d2dt_range(date_future))
#wr.check_sensors_flag = wr.check_sensors_flag or (wr_future.average_water_content <= 0.00)
if ( wr_future.average_water_content <= 0.00 ) and ( wr_future.do_not_irrigate == False ):
wr.check_sensors_flag = True
wr.days_to_irrigation = (date_future - date).days
break
except ObjectDoesNotExist:
pass
else: # execute below if irrigate_to_max_flag_seen is true
if wr.irrigate_to_max_achieved or irrigate_to_max_achieved or irrigate_to_max_days >= 3:
wr.irrigate_flag = False
wr.dry_down_flag = True
else:
irrigate_to_max_days += 1
if wr.average_water_content >= maxWater:
wr.irrigate_to_max_achieved = True
irrigate_to_max_achieved = True
wr.irrigate_flag = False
wr.check_sensors_flag = False
wr.dry_down_flag = True
else:
wr.irrigate_flag = True
wr.check_sensors_flag = True
## Write to the database
wr.save()
nChanged += 1
# reset the dependency date
field.earliest_changed_dependency_date = None
field.save()
return nChanged
def generate_objects(wh_formset, crop_season, field, user, report_date):
"""
Returns the list of object for display in the template.
Each object is an instance of the class UnifiedReport,
defined below.
We assume that the wh_formset is sorted by date, but not necessarily
by source. In this implementation there are two possible sources for
the WaterHistory objects for one date: at most one UGA and any number
of User.
"""
if crop_season.season_start_date > report_date:
return None
if crop_season.season_end_date < report_date:
report_date = crop_season.season_end_date
generate_water_register(crop_season, field, user, None, report_date)
water_register_query = WaterRegister.objects.filter(crop_season = crop_season, field = field)
water_register_query = water_register_query.order_by('-datetime')
water_register_query = water_register_query.filter(
Q(datetime__lte = d2dt_min(report_date + timedelta(WATER_REGISTER_DELTA)))
)
if len(water_register_query) == 0:
return None
### Get all the forms defined by the formset created from the water history objects
form_index = 0
current_form = None
all_forms = wh_formset.forms
ret = []
if all_forms is not None and len(all_forms) > 0:
current_form = all_forms[form_index]
form_index = form_index + 1
## Point to the first form in the current crop season
while current_form is not None and \
getDateObject(current_form['datetime'].value()) is not None and \
getDateObject(current_form['datetime'].value()) < crop_season.season_start_date:
if form_index == len(all_forms):
current_form = None
else:
print "IF WE SEE THIS THEN THIS LOOP IS NECESSARY"
current_form = all_forms[form_index]
form_index = form_index + 1
for day in daterange(crop_season.season_start_date, report_date + timedelta(days=1)):
try:
water_register = water_register_query.get(datetime__range = d2dt_range(day))
except:
print "Some date in crop season does not have a water register. Return nothing."
return None
day_record = UnifiedReport(day, water_register)
while current_form is not None and getDateObject(current_form['datetime'].value()) == day:
if current_form['source'].value() == "UGA":
if day_record.uga_form is not None:
message = "There is more than one UGA probe defined for %s on %s " % \
(crop_season, day)
#raise RuntimeError(message)
print message
day_record.add_uga(current_form)
elif current_form['source'].value() == "User":
day_record.all_forms.append(current_form)
else:
# raise RuntimeError("Unrecogized source type: " + current_form['source'].value())
print "Have a WaterHistory of 'Unknown' type: ", current_form
day_record.all_forms.append(current_form)
if form_index == len(all_forms):
current_form = None
else:
current_form = all_forms[form_index]
form_index = form_index + 1
ret.append(day_record)
# Add records for days in the future
### Might want days=WATER_REGISTER_DELTA+1 below, but we don't do it
### in generate_water_register
### Also this loop could be merged with above. But this is easier to see
### what happens
report_plus_delta = min(report_date + timedelta(days=WATER_REGISTER_DELTA), crop_season.season_end_date)
for day in daterange(report_date + timedelta(days=1), report_plus_delta):
water_register = water_register_query.filter(datetime__range = d2dt_range(day))
if len(water_register) == 1:
day_record = UnifiedReport(day, water_register[0])
ret.append(day_record)
else:
print "Found %d WaterRegister entries on %s for %s " % ( len(water_register),
day,
crop_season )
return ret
def courses(self, request, start, end):
"""
/courses/:start/:end
QUERY PARAMS
------------
include: ','-separated list of 'deleted', 'added', 'rescheduled', 'offdays'
Choose what types of mutation-affected course to show.
Unaffected courses are always included.
Courses are marked as such with the `mutation` attribute
Default: added,rescheduled,deleted
week-type: ','-separated list of 'Q1', 'Q2' | 'auto'
Choose which week types to include.
'auto': only shows the current week type.
Default: Q1
TODO(beta-1.1.0): handle /courses/:start
TODO(beta-1.1.0): handle ?week-type=auto
MAYBE(beta-1.1.0): use ?start & ?end instead of url fragments
"""
user = request.user
#
# Query params processing
#
# Get the date format and parse accordingly
def parse_iso8601(value):
fmt = 'datetime' if 'T' in value else 'date'
if '/' in value:
value = datetime.datetime.strptime(value, '%d/%m/%Y')
else:
value = datetime.datetime.fromisoformat(value)
return (fmt, value)
# Processing start & end
start_fmt, start = parse_iso8601(start)
end_fmt, end = parse_iso8601(end)
if start_fmt != end_fmt:
return Response({
'error':
'Date range bounds must have the same format',
})
using_time = start_fmt == 'datetime' # We could have picked end_fmt since start_fmt == end_fmt
# Processing include
include = request.query_params.get('include',
'added,rescheduled,deleted')
include = [i.strip() for i in include.split(',')]
# Translating query parameter's mutation types into internal ones used by .models.Mutation
mutation_types_map = {
'deleted': 'DEL',
'rescheduled': 'RES',
'added': 'ADD'
}
include = [mutation_types_map.get(i, i) for i in include]
# Processing week-type
week_types = request.query_params.get('week-type', 'Q1')
week_types = [w.strip() for w in week_types.split(',')]
#
# Additionnal data processing
#
# Getting offdays as date ranges
# Get the raw setting
try:
offdays = Setting.objects.get(setting__key='offdays').value
except Setting.DoesNotExist:
offdays = []
else:
# Each line is a new daterange/date
offdays = offdays.split('\n')
# Split to get start & end
offdays = [o.split(' - ') for o in offdays]
# For simple dates, set start & end to the same date
offdays = [[d[0], d[0]] for d in offdays if len(d) == 1]
# Parse the offdays' start & end dates into datetime.date objects
offdays = [[parse_iso8601(d.strip())[1] for d in o]
for o in offdays]
#
# Collecting events
#
# Init the variable containing all the courses
courses = []
# Loop over each day of the range
for day in daterange(start, end):
# TODO: handle ADD mutations
# Get the relevant events
events = Event.objects.filter(
Q(week_type__in=week_types) | Q(week_type='BOTH'),
Q(day=int(day.strftime('%u'))), Q(subject__user__id=user.id))
# Set the date part of each event, instead of a HH:MM time. (for start & end)
for event in events:
# Combine the date parts and the time parts
event.start = datetime.datetime.combine(
day.date(), # The loop's day (date part)
event.start # The event's start time (time part)
)
# Do the same for the end
event.end = datetime.datetime.combine(day.date(), event.end)
# Check if the event is in offdays
event.is_offday = False
# For each offday daterange
for offday in offdays:
offday_start, offday_end = offday
# Check if the event is in it
# If its already an offday because of another offday daterange, no need to check.
if not event.is_offday:
event.is_offday = dateranges_overlap(
(event.start, event.end),
(offday_start, offday_end))
# Don't add the course to the list if the conditions aren't met
if event.is_offday and 'offdays' not in include:
continue
# Check if the event matches a mutation
# (for EDT/RES/DEL mutations)
# for ADD mutations, the mutation isn't bounded to an event.
# We treat this separetly, and add them to the courses list outside of the events loop
event.mutation = None
for mutation in event.mutations.all():
# Determine if the mutation's date ranges matches the event
deleted_matches = False
added_matches = False
# Mutation types: see models.Mutation
if mutation.type in ('DEL', 'RES'):
deleted_matches = dateranges_overlap(
(mutation.deleted_start, mutation.deleted_end),
(event.start, event.end))
elif mutation.type == 'RES':
added_matches = dateranges_overlap(
(mutation.added_start, mutation.added_end),
(event.start, event.end))
# There should be at most one mutation relating to a single _course_ (not event).
# We set the mutation without further checking.
event.mutation = mutation
event.original_room = event.room
if event.mutation is not None:
# Process EDT/RES mutations that could modify the room
if event.mutation.type in (
'EDT', 'RES') and event.mutation.room is not None:
event.room = event.mutation.room
# Don't append courses whose mutation's type is not in include
if event.mutation.type not in include:
continue
# Append the course to the list
courses.append(event)
# Return the response
return Response(
[CourseReadSerializer(course).data for course in courses])