def readEvent(self):
mongoDB = MongoDBInterface('grande', 27017)
#mongoDB.SetDB('alarm_filter')
#mongoDB.SetCollection('photos')
mongoDB.SetDB('citybeat')
mongoDB.SetCollection('candidate_event_25by25_merged')
self.events = mongoDB.GetAllItems()
def ProcessRawEvents(rawEvent):
myDB = MongoDBInterface('grande', 27017)
myDB.SetDB('citybeat')
myDB.SetCollection('photos')
event = {}
photos = []
lineNumber = -1
for line in rawEvent:
lineNumber = lineNumber + 1
if lineNumber == 0:
event['created_time'] = line.strip()
continue
if lineNumber == 1:
event['lat'] = line.split(',')[0].strip()
event['lng'] = line.split(',')[1].strip()
continue
if lineNumber == 2:
vals = line.split(' ')
event['predicted_mu'] = vals[0].strip()
event['predicted_std'] = vals[1].strip()
event['within_range'] = vals[4].strip()
event['actual_value'] = vals[7].strip()
continue
photo_id = line.split(' ')[5].strip()
photo = myDB.GetItem({'id':photo_id})
photos.append(photo)
event['photos'] = photos
return event
def QueryFromDB():
mdbi = MongoDBInterface('grande', 27017)
mdbi.SetDB('historic_alarm')
mdbi.SetCollection('raw_event')
lat = '40.750542'
lng = '-73.9931535'
conditions = {'lat': lat, 'lng': lng}
events = mdbi.GetAllItems(conditions)
i = 0
numberOfPhotos = []
for event in events:
i = i + 1
numberOfPhotos.append(len(event['photos']))
print len(event['photos'])
print event['created_time']
def ReadAlarmsFromFile(fileName):
file = open(fileName)
adi = AlarmDataInterface('grande', 27017, 'historic_alarm', 'raw_event')
mdbi = MongoDBInterface('grande', 27017)
mdbi.SetDB('historic_alarm')
mdbi.SetCollection('raw_event')
event = []
for line in file:
if not line == '\n':
event.append(line)
else:
if not event == []:
print 'find an event'
# in json
processedEvent = ProcessRawEvents(event)
merged = adi.MergeEvent(processedEvent)
if not merged:
mdbi.SaveItem(processedEvent)
event = []
def main():
# inintialize the database for storing the alarms
myDB = MongoDBInterface('grande', 27017)
myDB.SetDB('alarm_filter')
myDB.SetCollection('photos')
adi = AlarmDataInterface()
regions = read_regions()
for region in regions:
to_save = {
'mid_lat': region[0],
'mid_lng': region[1],
'valid': False,
'real_count': -1,
'mu': -1,
'std': -1,
'photos': [],
'zscore': -1
}
photos = []
try:
ts, photos = find_photos_given_region(region[0], region[1],
'15Min', 'tmp_citybeat',
True)
except Exception as e:
save_to_mongo(to_save)
continue
sz = len(ts)
cur_value = 0
if sz > 0:
cur_time = ts.index[len(ts.index) - 1]
if isnull(ts.values[len(ts.values) - 1]):
cur_value = -1
else:
cur_value = ts.values[len(ts.values) - 1]
else:
save_to_mongo(to_save)
continue
if total_seconds(datetime.utcnow() - cur_time) > 900:
# there's no latest photo at this region
save_to_mongo(to_save)
continue
try:
predictions = get_prediction(str(region[0]), str(region[1]))
except Exception as e:
save_to_mongo(to_save)
continue
predict = -1
for p in predictions:
seconds = total_seconds(p['time'] - cur_time)
if (seconds <= 3600 and seconds >= 0):
#print 'p time is ',p['time'], 'cur_time is ',cur_time
predict = p
if predict == -1:
save_to_mongo(to_save)
continue
mu = float(predict['mu']) / 4.0
std = sqrt(float(predict['var'])) / 4.0
zscore = (cur_value - mu) / std
within_range = float(predict['mu']) / 4.0 + 3 * sqrt(
float(predict['var'])) / 4.0
photos_to_save = []
for photo in photos:
delta = total_seconds(
cur_time -
datetime.utcfromtimestamp(float(photo['created_time'])))
if delta >= 0 and delta <= 900:
photo['label'] = 'unlabeled'
photos_to_save.append(photo)
to_save['photos'] = photos_to_save
to_save['real_count'] = cur_value
to_save['valid'] = True
to_save['std'] = sqrt(float(predict['var'])) / 4.0
to_save['mu'] = float(predict['mu']) / 4.0
to_save['zscore'] = zscore
save_to_mongo(to_save)
if zscore >= 2.0 and cur_value >= 3:
print datetime.utcnow()
print region[0], ",", region[1]
print float(predict['mu']) / 4.0, sqrt(
float(predict['var']
)) / 4.0, 'range ', within_range, 'real-> ', cur_value
for photo in photos_to_save:
print 'printing photos: ', photo['link'], photo[
'created_time'], photo['id'], datetime.utcfromtimestamp(
float(photo['created_time']))
#group photos into an unlabeled event and save it to the db
predicted_mu = float(predict['mu']) / 4.0
predicted_std = sqrt(float(predict['var'])) / 4.0
newEvent = {
'created_time': datetime.utcnow(),
'mid_lat': region[0],
'mid_lng': region[1],
'predicted_mu': predicted_mu,
'predicted_std': predicted_std,
'actual_value': cur_value,
'zscore': zscore,
'photos': photos_to_save,
'label': 'unlabeled'
}
if not adi.MergeEvent(newEvent):
print 'created an event'
myDB.SaveItem(newEvent)
else:
print 'merged an event'
print '\n'
class AlarmDataInterface:
def __init__(self,
address=dbAddress,
port=27017,
db='alarm_filter',
collection='photos'):
self.db = MongoDBInterface(address, port)
self.db.SetDB(db)
self.db.SetCollection(collection)
def GetUnlabeledEvent(self):
# Get unlabeled "event" from the alarm filter
return self.db.GetItem({'label': 'unlabeled'})
def LabelEvent(self, event, label):
# Label an "event" as a true event or non-event
event['label'] = label
self.db.UpdateItem(event)
def _GetAllEventsAtLocation(self, lat, lon):
return self.db.GetAllItems({'mid_lat': lat, 'mid_lng': lon})
def _MergeTwoEvents(self, oldEvent, newEvent):
# merge the photos from the new event to the old event
# it will remove the duplicate photos
oldPhotos = oldEvent['photos']
newPhotos = newEvent['photos']
oldPhoto = oldPhotos[-1]
for i in xrange(0, len(newPhotos)):
if self._UnicodeToInt(
newPhotos[i]['created_time']) > self._UnicodeToInt(
oldPhoto['created_time']):
oldPhotos = oldPhotos + newPhotos[i:]
print '%d out of %d photos have been increased' % (len(
newPhotos[i:]), len(newPhotos))
break
# this print is just for debug
print 'no photo has been increased'
# end for debug
oldEvent['photos'] = oldPhotos
return oldEvent
def _UnicodeToInt(self, unic):
return string.atoi(unic.encode("utf-8"))
def MergeEvent(self, event):
# merge the event with older events in DB
# get all events in the same location
allEvents = self._GetAllEventsAtLocation(event['mid_lat'],
event['mid_lng'])
if allEvents is None:
return False
for oldEvent in allEvents:
# find a proper old event to combine (we assume there is only one "proper" old event)
lastPhoto = oldEvent['photos'][-1]
t1 = self._UnicodeToInt(event['photos'][-1]['created_time'])
t2 = self._UnicodeToInt(lastPhoto['created_time'])
if t1 == t2:
# no further photos for this event
return True
# maximal allowed time interval is 15 mins
if t1 > t2 and t1 <= t2 + 60 * 15:
#within 15 minutes
mergedEvent = self._MergeTwoEvents(oldEvent, event)
self.db.UpdateItem(mergedEvent)
return True
return False
def GetUnlabelEvents(self, condition=None):
# TODO: select events according to the conditions
pass