def main():
data_source = "instagram"
# data_source = 'twitter'
predict_days = 1
regions = read_regions()
redis_conn = Redis("tall4")
q = Queue(connection=redis_conn)
cnt = 0
async_results = {}
model_update_time = datetime.utcnow()
for region in regions:
par = cnt
if data_source == "twitter":
pass
try:
print region
ts, tweets = find_tweets_given_region(region[0], region[1], "1h", "tweets", True)
except Exception as e:
print e
continue
elif data_source == "instagram":
try:
ts, photos = find_photos_given_region(region[0], region[1], "1h", "citybeat", True)
except Exception as e:
print e
continue
start, training = process_ts(ts)
testing, align = get_testing(model_update_time, start, predict_days)
print "start is ", start
print "model_update_time is ", model_update_time
print "testing is ", testing
async_results[cnt] = q.enqueue_call(Predict, args=(training, testing, cnt), timeout=1720000, result_ttl=-1)
cnt += 1
done = False
begin_time = time.time()
time.sleep(2)
saved_flag = [0] * len(async_results)
while not done:
print "Time elapsed : ", time.time() - begin_time
done = True
for x in range(cnt):
result = async_results[x].return_value
if result is None:
done = False
continue
if saved_flag[x] == 0:
result = do_align(align, result)
if data_source == "twitter":
save_to_mongo(result, regions[x], model_update_time, "twitter")
elif data_source == "instagram":
print "work here"
save_to_mongo(result, regions[x], model_update_time, "instagram")
saved_flag[x] = 1
time.sleep(20)
def main():
data_source = 'instagram'
#data_source = 'twitter'
predict_days = 1
regions = read_regions()
redis_conn = Redis('tall4')
q = Queue(connection=redis_conn)
cnt = 0
async_results = {}
model_update_time = datetime.utcnow()
for region in regions:
par = cnt
if data_source=='twitter':
pass
try:
print region
ts, tweets = find_tweets_given_region(region[0], region[1], '1h','tweets',True)
except Exception as e:
print e
continue
elif data_source =='instagram':
try:
ts, photos = find_photos_given_region(region[0], region[1], '1h','citybeat',True)
except Exception as e:
print e
continue
start, training = process_ts(ts)
testing, align= get_testing(model_update_time, start, predict_days)
print 'start is ',start
print 'model_update_time is ',model_update_time
print 'testing is ',testing
async_results[cnt] = q.enqueue_call( Predict, args = ( training,testing, cnt,), timeout=1720000, result_ttl=-1 )
cnt+=1
done = False
begin_time = time.time()
time.sleep(2)
saved_flag = [0]*len(async_results)
while not done:
print "Time elapsed : ",time.time()-begin_time
done = True
for x in range(cnt):
result = async_results[x].return_value
if result is None:
done = False
continue
if saved_flag[x] == 0:
result = do_align(align, result)
if data_source=='twitter':
save_to_mongo(result, regions[x], model_update_time, "twitter")
elif data_source=='instagram':
print 'work here'
save_to_mongo(result, regions[x], model_update_time, "instagram")
saved_flag[x] = 1
time.sleep(20)
def main():
predict_days = 1
regions = read_regions()
redis_conn = Redis('tall4')
q = Queue(connection=redis_conn)
cnt = 0
async_results = {}
start_time = []
model_update_time = datetime.utcnow()
for region in regions:
par = cnt
try:
ts = find_photos_given_region(region[0], region[1])
except Exception as e:
print e
continue
start, training = process_ts(ts)
start_time.append(start)
#testing = get_testing(model_update_time, ( ts.index[len(ts)-1] - start).days, predict_days)
testing, align= get_testing(model_update_time, start, predict_days)
print 'start is ',start
print 'model_update_time is ',model_update_time
print 'testing is ',testing
async_results[cnt] = q.enqueue_call( Predict, args = ( training,testing, cnt,), timeout=1720000, result_ttl=-1 )
# Only for temporal test
fileName = '/grad/users/kx19/xia/test_tmp/training_time_' + str(cnt) + '.txt'
fout = open(fileName, 'w')
fout.write('Before Distributed GP:\n')
for t in testing:
fout.write(str(t))
fout.write('\n')
fout.close()
# end for test
cnt+=1
# Only for temporal test use
break
# end for test
done = False
begin_time = time.time()
time.sleep(2)
saved_flag = [0]*len(async_results)
while not done:
print "Time elapsed : ",time.time()-begin_time
done = True
for x in range(cnt):
#print 'checking ',x
result = async_results[x].return_value
#print 'check done'
#print 'res is ',result
if result is None:
done = False
continue
if saved_flag[x] == 0:
#result = fix_time(start_time[x], result)
result = do_align(align, result)
save_to_mongo(result, regions[x], model_update_time)
saved_flag[x] = 1
# Only for temporal test
fileName = '/grad/users/kx19/xia/test_tmp/training_time_' +str(x) + '.txt'
fout = open(fileName, 'a')
fout.write('After Test:\n')
for i in xrange(len(result)):
fout.write(str(result[i][0]))
fout.write('\n')
fout.close()
# end for test
time.sleep(0.2)
def main():
# inintialize the database for storing the alarms
myDB = MongoDBInterface('grande', 27017)
myDB.SetDB('alarm_filter')
myDB.SetCollection('photos')
adi = AlarmDataInterface()
regions = read_regions()
for region in regions:
to_save = {
'mid_lat': region[0],
'mid_lng': region[1],
'valid': False,
'real_count': -1,
'mu': -1,
'std': -1,
'photos': [],
'zscore': -1
}
photos = []
try:
ts, photos = find_photos_given_region(region[0], region[1],
'15Min', 'tmp_citybeat',
True)
except Exception as e:
save_to_mongo(to_save)
continue
sz = len(ts)
cur_value = 0
if sz > 0:
cur_time = ts.index[len(ts.index) - 1]
if isnull(ts.values[len(ts.values) - 1]):
cur_value = -1
else:
cur_value = ts.values[len(ts.values) - 1]
else:
save_to_mongo(to_save)
continue
if total_seconds(datetime.utcnow() - cur_time) > 900:
# there's no latest photo at this region
save_to_mongo(to_save)
continue
try:
predictions = get_prediction(str(region[0]), str(region[1]))
except Exception as e:
save_to_mongo(to_save)
continue
predict = -1
for p in predictions:
seconds = total_seconds(p['time'] - cur_time)
if (seconds <= 3600 and seconds >= 0):
#print 'p time is ',p['time'], 'cur_time is ',cur_time
predict = p
if predict == -1:
save_to_mongo(to_save)
continue
mu = float(predict['mu']) / 4.0
std = sqrt(float(predict['var'])) / 4.0
zscore = (cur_value - mu) / std
within_range = float(predict['mu']) / 4.0 + 3 * sqrt(
float(predict['var'])) / 4.0
photos_to_save = []
for photo in photos:
delta = total_seconds(
cur_time -
datetime.utcfromtimestamp(float(photo['created_time'])))
if delta >= 0 and delta <= 900:
photo['label'] = 'unlabeled'
photos_to_save.append(photo)
to_save['photos'] = photos_to_save
to_save['real_count'] = cur_value
to_save['valid'] = True
to_save['std'] = sqrt(float(predict['var'])) / 4.0
to_save['mu'] = float(predict['mu']) / 4.0
to_save['zscore'] = zscore
save_to_mongo(to_save)
if zscore >= 2.0 and cur_value >= 3:
print datetime.utcnow()
print region[0], ",", region[1]
print float(predict['mu']) / 4.0, sqrt(
float(predict['var']
)) / 4.0, 'range ', within_range, 'real-> ', cur_value
for photo in photos_to_save:
print 'printing photos: ', photo['link'], photo[
'created_time'], photo['id'], datetime.utcfromtimestamp(
float(photo['created_time']))
#group photos into an unlabeled event and save it to the db
predicted_mu = float(predict['mu']) / 4.0
predicted_std = sqrt(float(predict['var'])) / 4.0
newEvent = {
'created_time': datetime.utcnow(),
'mid_lat': region[0],
'mid_lng': region[1],
'predicted_mu': predicted_mu,
'predicted_std': predicted_std,
'actual_value': cur_value,
'zscore': zscore,
'photos': photos_to_save,
'label': 'unlabeled'
}
if not adi.MergeEvent(newEvent):
print 'created an event'
myDB.SaveItem(newEvent)
else:
print 'merged an event'
print '\n'
def main():
# inintialize the database for storing the alarms
myDB = MongoDBInterface('grande', 27017)
myDB.SetDB('alarm_filter')
myDB.SetCollection('photos')
regions = read_regions()
for region in regions:
to_save = {'mid_lat':region[0], 'mid_lng':region[1], 'valid':False, 'real_count':-1,'mu':-1, 'std':-1, 'photos':[], 'zscore':-1};
photos = []
try:
ts, photos = find_photos_given_region(region[0], region[1], '15Min', 'tmp_citybeat', True)
except Exception as e:
save_to_mongo(to_save)
continue
sz = len(ts)
cur_value = 0
if sz>0:
cur_time = ts.index[len(ts.index)-1]
if isnull(ts.values[len(ts.values)-1]):
cur_value = -1
else:
cur_value = ts.values[len(ts.values)-1]
else:
save_to_mongo(to_save)
continue
if total_seconds(datetime.utcnow() - cur_time)>900:
# there's no latest photo at this region
save_to_mongo(to_save)
continue
try:
predictions = get_prediction(str(region[0]), str(region[1]))
except Exception as e:
save_to_mongo(to_save)
continue
predict = -1
for p in predictions:
seconds = total_seconds( p['time']-cur_time)
if( seconds<=3600 and seconds>=0):
#print 'p time is ',p['time'], 'cur_time is ',cur_time
predict = p
if predict==-1:
save_to_mongo(to_save)
continue
mu = float(predict['mu'])/4.0
std = sqrt(float(predict['var']))/4.0
zscore = (cur_value - mu)/std
within_range = float(predict['mu'])/4.0+3*sqrt(float(predict['var']))/4.0
photos_to_save = []
for photo in photos:
delta = total_seconds( cur_time - datetime.utcfromtimestamp( float(photo['created_time'])) )
if delta>=0 and delta<=900:
photos_to_save.append(photo)
to_save['photos'] = photos_to_save
to_save['real_count'] = cur_value
to_save['valid'] = True
to_save['std'] = sqrt(float(predict['var']))/4.0
to_save['mu'] = float(predict['mu'])/4.0
to_save['zscore'] = zscore
save_to_mongo(to_save)
if zscore>=3 and cur_value>=8:
print datetime.utcnow()
print region[0],",",region[1]
print float(predict['mu'])/4.0, sqrt(float(predict['var']))/4.0, 'range ',within_range,'real-> ',cur_value
for photo in photos_to_save:
print 'printing photos: ', photo['link'], photo['created_time'], photo['id'], datetime.utcfromtimestamp(float(photo['created_time']))
print '\n'
#group photos into an unlabeled event and save it to the db
predicted_mu = float(predict['mu'])/4.0
predicted_std = sqrt(float(predict['var']))/4.0
event = {'discovered_time':datetime.utcnow(), 'lat':region[0], 'lng':region[1], 'predicted_mu':predicted_mu, 'predicted_std':predicted_std, 'actual_value':cur_value, 'zscore':zscore, 'photos':photos_to_save, 'label':'unlabeled'}
myDB.SaveItem(event)
def main():
predict_days = 1
regions = read_regions()
redis_conn = Redis('tall4')
q = Queue(connection=redis_conn)
cnt = 0
async_results = {}
start_time = []
model_update_time = datetime.utcnow()
for region in regions:
par = cnt
try:
ts = find_photos_given_region(region[0], region[1])
except Exception as e:
print e
continue
start, training = process_ts(ts)
start_time.append(start)
#testing = get_testing(model_update_time, ( ts.index[len(ts)-1] - start).days, predict_days)
testing, align = get_testing(model_update_time, start, predict_days)
print 'start is ', start
print 'model_update_time is ', model_update_time
print 'testing is ', testing
async_results[cnt] = q.enqueue_call(Predict,
args=(
training,
testing,
cnt,
),
timeout=1720000,
result_ttl=-1)
# Only for temporal test
fileName = '/grad/users/kx19/xia/test_tmp/training_time_' + str(
cnt) + '.txt'
fout = open(fileName, 'w')
fout.write('Before Distributed GP:\n')
for t in testing:
fout.write(str(t))
fout.write('\n')
fout.close()
# end for test
cnt += 1
# Only for temporal test use
break
# end for test
done = False
begin_time = time.time()
time.sleep(2)
saved_flag = [0] * len(async_results)
while not done:
print "Time elapsed : ", time.time() - begin_time
done = True
for x in range(cnt):
#print 'checking ',x
result = async_results[x].return_value
#print 'check done'
#print 'res is ',result
if result is None:
done = False
continue
if saved_flag[x] == 0:
#result = fix_time(start_time[x], result)
result = do_align(align, result)
save_to_mongo(result, regions[x], model_update_time)
saved_flag[x] = 1
# Only for temporal test
fileName = '/grad/users/kx19/xia/test_tmp/training_time_' + str(
x) + '.txt'
fout = open(fileName, 'a')
fout.write('After Test:\n')
for i in xrange(len(result)):
fout.write(str(result[i][0]))
fout.write('\n')
fout.close()
# end for test
time.sleep(0.2)