def handle(self, text):
# expected format:
#
# recieved books 123450 4 1
# | | | | | |
# V | | | | |
# handler V | | | |
# commodity | | | |
# V | | |
# school code | | |
# V | |
# satellite # | |
# V |
# # of units |
# V
# condition code
# declare variables intended for valid information
known_contact = None
commodity = None
facility = None
quantity = None
condition = None
observed_cargo = None
if self.msg.contact is not None:
self.debug(self.msg.contact)
known_contact = self.msg.contact
elif self.msg.connection.identity is not None:
try:
known_contact = Contact.objects.get(phone=self.msg.connection.identity)
self.msg.connection.contact = known_contact
self.msg.connection.save()
except MultipleObjectsReturned:
#TODO do something?
self.debug('MULTIPLE IDENTITIES')
pass
except ObjectDoesNotExist:
self.debug('NO PERSON FOUND')
try:
known_contact = Contact.objects.get(alternate_phone=\
self.msg.connection.identity)
self.msg.connection.contact = known_contact
self.msg.connection.save()
except MultipleObjectsReturned:
#TODO this case may be unneccesary, since many many contacts
# often share a single alternate_phone
self.debug('MULTIPLE IDENTITIES AFTER UNKNOWN')
pass
except ObjectDoesNotExist:
#self.respond("Sorry, I don't recognize your phone number. Please respond with your surname, facility (school or DEO) code, and facility name.")
pass
finally:
known_contact = Contact.objects.create(phone=self.msg.connection.identity)
else:
self.debug('NO IDENTITY')
if known_contact is not None:
self.debug('KNOWN PERSON')
expected_tokens = ['word', 'number', 'number', 'number']
token_labels = ['commodity', 'school_code', 'quantity', 'condition']
tokens = utils.split_into_tokens(expected_tokens, token_labels, text)
self.debug(tokens)
if not tokens['commodity'].isdigit():
def get_commodity(token):
try:
# lookup commodity by slug
com = Commodity.objects.get(slug__istartswith=tokens['commodity'])
return com
except MultipleObjectsReturned:
#TODO do something here?
pass
except ObjectDoesNotExist:
coms = Commodity.objects.all()
for com in coms:
# iterate all commodities and see if submitted
# token is in an aliases list
match = com.has_alias(token)
if match is not None:
if match:
return com
continue
return None
commodity = get_commodity(tokens['commodity'])
if commodity is None:
self.respond("Sorry, no supply called '%s'" % (tokens['commodity']))
self.respond("Approved supplies are %s" % ", ".join(Commodity.objects.values_list('slug', flat=True)))
if tokens['school_code'].isdigit():
def list_possible_schools_for_code(school_num):
possible_schools = School.objects.filter(code=school_num)
if not possible_schools:
return None
else:
# format a list containing
# 1) combined school code + satellite_number
# 2) school name in parentheses
clean_list = []
for school in possible_schools:
clean_list.append(school.full_code + " (" + school.name + ")")
return clean_list
# school code should be between 1 and 5 digits,
# and satellite_number should be 1 digit.
# in the interest of not hardcoding anything, lets hit the db!
max_codes = School.objects.aggregate(max_code=Max('code'),\
max_sat=Max('satellite_number'))
max_code_length = len(str(max_codes['max_code']))
max_sat_length = len(str(max_codes['max_sat']))
if len(tokens['school_code']) <= (max_code_length + max_sat_length):
# separate school's code and satellite_number (last digit)
school_num = tokens['school_code'][:-1]
sat_num = tokens['school_code'][-1:]
try:
school = School.objects.get(code=school_num,\
satellite_number=sat_num)
facility, f_created = Facility.objects.get_or_create(location_id=school.pk,\
location_type=ContentType.objects.get(model='school'))
except ObjectDoesNotExist:
try:
school = School.objects.get(code=tokens['school_code'])
facility, f_created = Facility.objects.get_or_create(location_id=school.pk,\
location_type=ContentType.objects.get(model='school'))
except ObjectDoesNotExist:
self.respond("Sorry, cannot find school with code '%s'" % (tokens['school_code']))
# maybe satellite number is omitted, so lookup schools by entire token
suggestions = list_possible_schools_for_code(tokens['school_code'])
if suggestions is not None:
self.respond("Did you mean one of: %s?" %\
(", ".join(suggestions)))
# maybe satellite number is incorrect, so lookup schools only by school_code
suggestions = list_possible_schools_for_code(school_num)
if suggestions is not None:
self.respond("Did you mean one of: %s?" %\
(", ".join(suggestions)))
else:
self.respond("Sorry code '%s' is not valid. All codes are fewer than 6 digits" % (tokens['school_code']))
#TODO acceptible values should be configurable
#if int(tokens['quantity']) in range(1,10):
# if int(tokens['condition']) in range(1,4):
if tokens['quantity'].isdigit():
if tokens['condition'].isdigit():
# map expected condition tokens into choices for db
conditions_map = {'1':'G', '2':'D', '3':'L'}
if facility is not None:
active_shipment = Facility.get_active_shipment(facility)
if active_shipment is not None:
# create a new Cargo object
observed_cargo = Cargo.objects.create(\
commodity=commodity,\
quantity=int(tokens['quantity']),\
condition=conditions_map[tokens['condition']])
# create a new ShipmentSighting
sighting = ShipmentSighting.objects.create(\
observed_cargo=observed_cargo,\
facility=facility)
# associate new Cargo with Shipment
active_shipment.status = 'D'
active_shipment.actual_delivery_time=datetime.datetime.now()
active_shipment.cargos.add(observed_cargo)
active_shipment.save()
# get or create a ShipmentRoute and associate
# with new ShipmentSighting
route, new_route = ShipmentRoute.objects.get_or_create(\
shipment=active_shipment)
route.sightings.add(sighting)
route.save()
campaign = Campaign.get_active_campaign()
if campaign is not None:
campaign.shipments.add(active_shipment)
campaign.save()
data = [
"%s pallets" % (observed_cargo.quantity or "??"),
"of %s" % (commodity.slug or "??"),
"to %s" % (facility.location.name or "??"),
"in %s condition" % (observed_cargo.get_condition_display() or "??")
]
confirmation = "Thanks. Confirmed delivery of %s." %\
(" ".join(data))
self.respond(confirmation)
def handle(self, text):
# expected format:
#
# confirm books gwaai valley primary 4288 G
# | | | | |
# V | | | |
# handler V | | |
# commodity | | |
# V | |
# school name | |
# | |
# | |
# V |
# school code |
# |
# |
# V
# status
# declare variables intended for valid information
known_contact = None
commodity = None
quantity = None
condition = None
observed_cargo = None
facility = None
school = None
possible_schools = []
possible_by_code = None
possible_by_name = None
school_by_code = None
school_by_name = None
if self.msg.contact is not None:
self.debug(self.msg.contact)
known_contact = self.msg.contact
elif self.msg.connection.identity is not None:
try:
known_contact = Contact.objects.get(phone=self.msg.connection.identity)
self.msg.connection.contact = known_contact
self.msg.connection.save()
except MultipleObjectsReturned:
#TODO do something?
self.debug('MULTIPLE IDENTITIES')
pass
except ObjectDoesNotExist:
self.debug('NO PERSON FOUND')
try:
known_contact = Contact.objects.get(alternate_phone=\
self.msg.connection.identity)
self.msg.connection.contact = known_contact
self.msg.connection.save()
except MultipleObjectsReturned:
#TODO this case may be unneccesary, since many many contacts
# often share a single alternate_phone
self.debug('MULTIPLE IDENTITIES AFTER UNKNOWN')
pass
except ObjectDoesNotExist:
#self.respond("Sorry, I don't recognize your phone number. Please respond with your surname, facility (school or DEO) code, and facility name.")
pass
finally:
known_contact, new_contact = Contact.objects.get_or_create(phone=self.msg.connection.identity)
else:
self.debug('NO IDENTITY')
if known_contact is not None:
self.debug('KNOWN PERSON')
# lists of expected token types and their labels for split_into_tokens
expected_tokens = ['word', 'words', 'number', 'word']
token_labels = ['commodity', 'school_name', 'school_code', 'condition']
tokens = utils.split_into_tokens(expected_tokens, token_labels, text)
self.debug(tokens)
if not tokens['commodity'].isdigit():
def get_commodity(token):
try:
# lookup commodity by slug
com = Commodity.objects.get(slug__istartswith=tokens['commodity'])
return com
except MultipleObjectsReturned:
#TODO do something here?
pass
except ObjectDoesNotExist:
coms = Commodity.objects.all()
for com in coms:
# iterate all commodities and see if submitted
# token is in an aliases list
match = com.has_alias(token)
if match is not None:
if match:
return com
continue
return None
commodity = get_commodity(tokens['commodity'])
if commodity is None:
self.respond("Sorry, no supply called '%s'" % (tokens['commodity']))
self.respond("Approved supplies are %s" % ", ".join(Commodity.objects.values_list('slug', flat=True)))
if not tokens['school_name'].isdigit():
try:
# first try to match name exactly
school = School.objects.get(name__iexact=tokens['school_name'])
facility, f_created = Facility.objects.get_or_create(location_id=school.pk,\
location_type=ContentType.objects.get(model='school'))
except MultipleObjectsReturned:
# if there are many exact matches, add them to the possible_schools list
schools = School.objects.filter(name__istartswith=tokens['school_name'])
for school in schools:
possible_schools.append(school)
except ObjectDoesNotExist:
# try to match using string distance algorithms
possible_by_name = School.closest_by_spelling(tokens['school_name'])
self.debug("%s possible facilities by name" % (str(len(possible_by_name))))
self.debug(possible_by_name)
if len(possible_by_name) == 1:
if possible_by_name[0][2] == 0 and possible_by_name[0][3] == 0 and possible_by_name[0][4] == 1.0:
self.debug('PERFECT LOC MATCH BY NAME')
school_by_name = possible_by_name[0][1]
school = school_by_name
facility, f_created = Facility.objects.get_or_create(location_id=school.pk,\
location_type=ContentType.objects.get(model='school'))
else:
if possible_by_name is not None:
for fac in possible_by_name:
# add any non-perfect matches to possible_schools
possible_schools.append(fac[1])
if tokens['school_code'].isdigit():
possible_by_code = School.closest_by_code(tokens['school_code'])
self.debug("%s possible facilities by code" % (str(len(possible_by_code))))
if len(possible_by_code) == 1:
if possible_by_code[0][2] == 0 and possible_by_code[0][3] == 0 and possible_by_code[0][4] == 1.0:
self.debug('PERFECT LOC MATCH BY CODE')
school_by_code = possible_by_code[0][1]
# see if either facility lookup returned a perfect match
if school_by_code or school_by_name is not None:
if school_by_code and school_by_name is not None:
# if they are both the same perfect match we have a winner
if school_by_code.pk == school_by_name.pk:
school = school_by_code
facility, f_created = Facility.objects.get_or_create(location_id=school.pk,\
location_type=ContentType.objects.get(model='school'))
# if we have two different perfect matches, add to list
else:
possible_schools.append(school_by_code)
self.debug("%s possible facilities" % (str(len(possible_schools))))
possible_facilities.append(school_by_name)
self.debug("%s possible facilities" % (str(len(possible_schools))))
else:
# perfect match by either is also considered a winner
school = school_by_code if school_by_code is not None else school_by_name
facility, f_created = Facility.objects.get_or_create(location_id=school.pk,\
location_type=ContentType.objects.get(model='school'))
self.debug(facility)
# neither lookup returned a perfect match
else:
# make list of facility objects that are in both fac_by_code and fac_by_name
if possible_by_code and possible_by_name is not None:
possible_schools.extend([l[1] for l in filter(lambda x:x in possible_by_code, possible_by_name)])
self.debug("%s possible facilities by both" % (str(len(possible_schools))))
if len(possible_schools) == 0:
possible_schools.extend([l[1] for l in possible_by_code if possible_by_code is not None])
possible_schools.extend([l[1] for l in possible_by_name if possible_by_name is not None])
self.debug("%s possible facilities by both" % (str(len(possible_schools))))
if len(possible_schools) == 1:
school = possible_schools[0]
facility, f_created = Facility.objects.get_or_create(location_id=school.pk,\
location_type=ContentType.objects.get(model='school'))
if facility is None:
self.respond("Sorry I don't know '%s'" % (tokens['school_name']))
self.respond("Did you mean one of: %s?" %\
(", ".join(possible_schools)))
if facility is not None:
if not tokens['condition'].isdigit():
if facility is not None:
active_shipment = Facility.get_active_shipment(facility)
if active_shipment is not None:
# create a new Cargo object
condition = tokens['condition'].upper()
if condition in ['G', 'D', 'L', 'I']:
observed_cargo = Cargo.objects.create(\
commodity=commodity,\
condition=condition)
else:
self.respond("Oops. Status must be one of: G, D, L, or I")
seen_by_str = self.msg.connection.backend.name + ":" + self.msg.connection.identity
# create a new ShipmentSighting
sighting = ShipmentSighting.objects.create(\
observed_cargo=observed_cargo,\
facility=facility, seen_by=seen_by_str)
# associate new Cargo with Shipment
active_shipment.status = 'D'
active_shipment.actual_delivery_time=datetime.datetime.now()
active_shipment.cargos.add(observed_cargo)
active_shipment.save()
# get or create a ShipmentRoute and associate
# with new ShipmentSighting
route, new_route = ShipmentRoute.objects.get_or_create(\
shipment=active_shipment)
route.sightings.add(sighting)
route.save()
if observed_cargo.condition is not None:
this_school = School.objects.get(pk=facility.location_id)
# map reported condition to the status numbers
# that the sparklines will use
map = {'G':1, 'D':-2, 'L':-3, 'I':-4}
if observed_cargo.condition in ['D', 'L', 'I', 'G']:
this_school.status = map[observed_cargo.condition]
else:
this_school.status = 0
this_school.save()
this_district = this_school.parent
# TODO optimize! this is very expensive
# and way too slow
# re-generate the list of statuses that
# the sparklines will use
updated = this_district.spark
campaign = Campaign.get_active_campaign()
if campaign is not None:
campaign.shipments.add(active_shipment)
campaign.save()
data = [
"of %s" % (commodity.slug or "??"),
"to %s" % (facility.location.name or "??"),
"in %s condition" % (observed_cargo.get_condition_display() or "??")
]
confirmation = "Thanks. Confirmed delivery of %s." %\
(" ".join(data))
self.respond(confirmation)
def go():
print datetime.datetime.now().isoformat()
clean_db = True
if clean_db:
schools = School.objects.all().update(status=0)
print "reset schools"
districts = District.objects.all().update(status=None)
print "reset districts"
shipments = Shipment.objects.all().update(status='P')
shipments = Shipment.objects.all().update(actual_delivery_time=None)
print "reset shipments"
confirmations = Confirmation.objects.all().delete()
print "deleted confirmations"
sightings = ShipmentSighting.objects.all().delete()
print "deleted sightings"
routes = ShipmentRoute.objects.all().delete()
print "deleted routes"
cargos = Cargo.objects.all().delete()
print "deleted cargos"
incoming = Message.objects.filter(direction='I')
unique_text = []
unique = []
# make list of unique incoming messages, based on the message text
for mess in incoming:
if mess.text not in unique_text:
unique_text.append(mess.text)
unique.append(mess)
# all school names, split into individual words,
# flattened into 1-d list, duplicates removed
school_name_words = list(set(list(itertools.chain.from_iterable([n.split() for n in School.objects.all().values_list('name', flat=True)]))))
# odd punctuation we want to get rid of
junk = ['.', ',', '\'', '\"', '`', '(', ')', ':', ';', '&', '?', '!', '~', '`', '+', '-']
school_name_words_no_punc = []
for mark in junk:
for word in school_name_words:
# remove punctuation from school name words, because we'll be
# removing the same punctuation from message text
school_name_words_no_punc.append(word.replace(mark, " "))
# if user has spelled out any of the conditions, we want to see those,
# as well as "L" -- other conditions "G", "D", "I" already appear in school_name_words_no_punc
other_words = ["INCOMPLETE", "GOOD", "DAMAGED", "ALTERNATE", "LOCATION", "L"]
ok_words = school_name_words_no_punc + other_words
print len(unique)
counter = 0
matches = 0
#for text in ['CONFIRM BOOKS DADATA PRIMARY 1196i']:
#for msg in unique[45:55]:
for msg in unique:
counter = counter + 1
if counter % 100 == 0:
print "loop: %s" % str(counter)
text = msg.text
text_list = []
# replace any creative punctuation with spaces
for mark in junk:
text = text.replace(mark, " ")
# split the text into chunks around spaces
blobs = text.split(" ")
for blob in blobs:
clean_blob = blob
try:
if blob[-1:].isalpha() and blob[:-1].isdigit():
# if theres somthing like '1234g'
# add as two separate blobs: '1234' and 'g'
text_list.append(blob[:-1])
text_list.append(blob[-1:])
# and move on to next blob before
# letters_for_numbers might duplicate it incorrectly
continue
except IndexError:
pass
for n in range(3):
# clean up blobs only if they have a digit in the first few
# characters -- so we don't clean up things like user1
try:
if blob[n].isdigit():
clean_blob = letters_for_numbers(blob)
break
except IndexError:
# if the blob doesnt have the first few characters,
# and there is no digit yet, move on
break
# add the cleaned blob (or untouched blob) to a running list
text_list.append(clean_blob)
relevant = []
# now, loop through cleaned words and keep relevant ones
for word in text_list:
if word.isdigit():
relevant.append(word)
continue
if word.upper() in ok_words:
relevant.append(word)
continue
# attach list of relevant bits to message
confirmation = Confirmation(message=msg)
confirmation.token_list = copy.copy(relevant)
confirmation.save()
# now try to make sense of these tokens
consumed = []
unconsumed = []
# generator to yield relevant items in reverse order
consume = consume_in_reverse(relevant)
condition = None
school = None
school_by_code = None
school_by_spelling = None
try:
def attempt_consumption_of_condition_and_code(condition, school_by_code):
token = consume.next()
if condition is None:
condition = reconcile_condition(token)
if condition is not None:
consumed.append(token)
else:
if token not in unconsumed:
unconsumed.append(token)
# if the last token (the first we have examined) this time
# has been consumed, pop the next-to-last token.
# otherwise, we will continue with the last token
if token in consumed:
token = consume.next()
# note this may be a school object or a list of tuples
# in the format:
# ('token', school_obj, lev_edit_int, dl_edit_int, jw_float)
if school_by_code is None:
school_by_code = reconcile_school_by_code(token)
if school_by_code is not None:
consumed.append(token)
confirmation.code = token
confirmation.save()
else:
if token not in unconsumed:
unconsumed.append(token)
if len(consumed) == 2:
return condition, school_by_code
else:
return attempt_consumption_of_condition_and_code(condition, school_by_code)
# recursively consume tokens until we have something for condition and school_by_code
condition, school_by_code = attempt_consumption_of_condition_and_code(condition, school_by_code)
if not isinstance(school_by_code, list):
# woo! we have a condition and a single school, this is probably
# enough to be sure about the school, so save it as school before
# exploding into finding the school name
school = school_by_code
confirmation.school = school
confirmation.save()
try:
school_name = None
# pop the next-to-next-to-last token
token = consume.next()
# now lets try to get the school name
if token in consumed:
token = consume.next()
school_name = token
consumed.append(token)
# consume up to five additional tokens and
# prepend to school_name
token = consume.next()
if token.isalpha():
school_name = token + " " + school_name
consumed.append(token)
else:
unconsumed.append(token)
token = consume.next()
if token.isalpha():
school_name = token + " " + school_name
consumed.append(token)
else:
unconsumed.append(token)
token = consume.next()
if token.isalpha():
school_name = token + " " + school_name
consumed.append(token)
else:
unconsumed.append(token)
token = consume.next()
if token.isalpha():
school_name = token + " " + school_name
consumed.append(token)
else:
unconsumed.append(token)
token = consume.next()
if token.isalpha():
school_name = token + " " + school_name
consumed.append(token)
else:
unconsumed.append(token)
except StopIteration:
if school_name is not None:
school_by_spelling = reconcile_school_by_spelling(school_name.strip())
p_schools = []
if isinstance(school_by_code, list):
for s in (t[1] for t in school_by_code):
if s not in p_schools:
p_schools.append(s)
if school_by_spelling is not None:
if not isinstance(school_by_spelling, list):
if school is not None:
if school.code == school_by_spelling.code:
pass
else:
p_schools.append(school_by_spelling)
else:
school = school_by_spelling
else:
for s in (t[1] for t in school_by_spelling):
if s is not None:
if s.code not in [p.code for p in p_schools if p is not None]:
p_schools.append(s)
else:
school = s
# if we have no sure match, and a list of possible schools
# returned by reconcile_school_by_spelling, try toggling
# the word primary
if school is None and isinstance(school_by_spelling, list):
uschool_name = school_name.upper()
if uschool_name.find("PRIMARY") != -1:
edited_name = uschool_name.replace("PRIMARY", "")
else:
edited_name = uschool_name + " PRIMARY"
school_by_spelling = reconcile_school_by_spelling(edited_name.strip())
if school_by_spelling is not None:
if not isinstance(school_by_spelling, list):
if school is not None:
if school.code == school_by_spelling.code:
pass
else:
p_schools.append(school_by_spelling)
else:
school = school_by_spelling
else:
for s in (t[1] for t in school_by_spelling):
if s.code not in [p.code for p in p_schools]:
p_schools.append(s)
else:
school = s
if school is None:
confirmation.possible_schools = [s.pk for s in p_schools]
confirmation.save()
if condition is not None:
confirmation.condition = condition
confirmation.save()
if school is not None:
if condition is not None:
confirmation.condition = condition
confirmation.valid = True
confirmation.save()
matches = matches + 1
if matches % 20 == 0:
print "MATCHES: %s out of %s" % (str(matches), str(counter))
print datetime.datetime.now().isoformat()
commodity = Commodity.objects.get(slug__istartswith="textbooks")
facility, f_created = Facility.objects.get_or_create(location_id=school.pk,\
location_type=ContentType.objects.get(model='school'))
if facility is not None:
active_shipment = Facility.get_active_shipment(facility)
observed_cargo = Cargo.objects.create(\
commodity=commodity,\
condition=condition)
seen_by_str = msg.connection.backend.name + ":" + msg.connection.identity
# create a new ShipmentSighting
sighting = ShipmentSighting.objects.create(\
observed_cargo=observed_cargo,\
facility=facility, seen_by=seen_by_str)
# associate new Cargo with Shipment
active_shipment.status = 'D'
active_shipment.actual_delivery_time=msg.date
active_shipment.cargos.add(observed_cargo)
active_shipment.save()
# get or create a ShipmentRoute and associate
# with new ShipmentSighting
route, new_route = ShipmentRoute.objects.get_or_create(\
shipment=active_shipment)
route.sightings.add(sighting)
route.save()
if observed_cargo.condition is not None:
this_school = School.objects.get(pk=facility.location_id)
# map reported condition to the status numbers
# that the sparklines will use
map = {'G':1, 'D':-2, 'L':-3, 'I':-4}
if observed_cargo.condition in ['D', 'L', 'I', 'G']:
this_school.status = map[observed_cargo.condition]
else:
this_school.status = 0
this_school.save()
this_district = this_school.parent
# TODO optimize! this is very expensive
# and way too slow
# re-generate the list of statuses that
# the sparklines will use
#updated = this_district.spark
campaign = Campaign.get_active_campaign()
if campaign is not None:
campaign.shipments.add(active_shipment)
campaign.save()
'''
data = [
"of %s" % (commodity.slug or "??"),
"to %s" % (facility.location.name or "??"),
"in %s condition" % (observed_cargo.get_condition_display() or "??")
]
confirmation = "Thanks. Confirmed delivery of %s." %\
(" ".join(data))
print seen_by_str + " " + confirmation
'''
except StopIteration:
continue
except Exception, e:
print e
print counter
print matches
import ipdb;ipdb.set_trace()