示例#1
0
def main():
    data_source = "instagram"
    # data_source = 'twitter'
    predict_days = 1
    regions = read_regions()
    redis_conn = Redis("tall4")
    q = Queue(connection=redis_conn)
    cnt = 0
    async_results = {}
    model_update_time = datetime.utcnow()

    for region in regions:
        par = cnt
        if data_source == "twitter":
            pass
            try:
                print region
                ts, tweets = find_tweets_given_region(region[0], region[1], "1h", "tweets", True)
            except Exception as e:
                print e
                continue
        elif data_source == "instagram":
            try:
                ts, photos = find_photos_given_region(region[0], region[1], "1h", "citybeat", True)
            except Exception as e:
                print e
                continue
        start, training = process_ts(ts)
        testing, align = get_testing(model_update_time, start, predict_days)
        print "start is ", start
        print "model_update_time is ", model_update_time
        print "testing is ", testing
        async_results[cnt] = q.enqueue_call(Predict, args=(training, testing, cnt), timeout=1720000, result_ttl=-1)
        cnt += 1
    done = False
    begin_time = time.time()
    time.sleep(2)
    saved_flag = [0] * len(async_results)
    while not done:
        print "Time elapsed : ", time.time() - begin_time
        done = True
        for x in range(cnt):
            result = async_results[x].return_value
            if result is None:
                done = False
                continue
            if saved_flag[x] == 0:
                result = do_align(align, result)
                if data_source == "twitter":
                    save_to_mongo(result, regions[x], model_update_time, "twitter")
                elif data_source == "instagram":
                    print "work here"
                    save_to_mongo(result, regions[x], model_update_time, "instagram")
                saved_flag[x] = 1
        time.sleep(20)
示例#2
0
def main():
    data_source = 'instagram'
    #data_source = 'twitter'
    predict_days = 1
    regions = read_regions()
    redis_conn = Redis('tall4')
    q = Queue(connection=redis_conn)
    cnt = 0
    async_results = {}
    model_update_time = datetime.utcnow()
    
    for region in regions:
        par = cnt
        if data_source=='twitter':
            pass
            try:
                print region
                ts, tweets = find_tweets_given_region(region[0], region[1], '1h','tweets',True)
            except Exception as e:
                print e
                continue
        elif data_source =='instagram':
            try:
                ts, photos = find_photos_given_region(region[0], region[1], '1h','citybeat',True)
            except Exception as e:
                print e
                continue
        start, training = process_ts(ts)
        testing, align= get_testing(model_update_time, start, predict_days)
        print 'start is ',start
        print 'model_update_time is ',model_update_time
        print 'testing is ',testing
        async_results[cnt] = q.enqueue_call( Predict, args = ( training,testing, cnt,), timeout=1720000, result_ttl=-1 )
        cnt+=1
    done = False
    begin_time = time.time()
    time.sleep(2)
    saved_flag = [0]*len(async_results)
    while not done:
        print "Time elapsed : ",time.time()-begin_time
        done = True
        for x in range(cnt):
            result = async_results[x].return_value
            if result is None:
                done = False
                continue
            if saved_flag[x] == 0:
                result = do_align(align, result)
                if data_source=='twitter':
                    save_to_mongo(result, regions[x], model_update_time, "twitter")
                elif data_source=='instagram':
                    print 'work here'
                    save_to_mongo(result, regions[x], model_update_time, "instagram")
                saved_flag[x] = 1
        time.sleep(20)
def main():
    predict_days = 1
    regions = read_regions()
    redis_conn = Redis('tall4')
    q = Queue(connection=redis_conn)
    cnt = 0
    async_results = {}
    start_time = []
    model_update_time = datetime.utcnow()
    
    for region in regions:
        par = cnt
        try:
            ts = find_photos_given_region(region[0], region[1])
        except Exception as e:
            print e
            continue
        start, training = process_ts(ts)
        start_time.append(start)
        #testing = get_testing(model_update_time, ( ts.index[len(ts)-1] - start).days, predict_days)
        testing, align= get_testing(model_update_time, start, predict_days)
        print 'start is ',start
        print 'model_update_time is ',model_update_time
        print 'testing is ',testing
        async_results[cnt] = q.enqueue_call( Predict, args = ( training,testing, cnt,), timeout=1720000, result_ttl=-1 )
        # Only for temporal test
        fileName = '/grad/users/kx19/xia/test_tmp/training_time_' + str(cnt) + '.txt'
        fout = open(fileName, 'w')
        fout.write('Before Distributed GP:\n')
        for t in testing:
            fout.write(str(t))
            fout.write('\n')
        fout.close()
        # end for test
        cnt+=1
        # Only for temporal test use
        break
        # end for test
    
    done = False
    begin_time = time.time()
    time.sleep(2)
    saved_flag = [0]*len(async_results)
    while not done:
        print "Time elapsed : ",time.time()-begin_time
        done = True
        for x in range(cnt):
            #print 'checking ',x
            result = async_results[x].return_value
            #print 'check done'
            #print 'res is ',result
            if result is None:
                done = False
                continue
            if saved_flag[x] == 0:
                #result = fix_time(start_time[x], result) 
                result = do_align(align, result)
                save_to_mongo(result, regions[x], model_update_time)
                saved_flag[x] = 1
                # Only for temporal test
                fileName = '/grad/users/kx19/xia/test_tmp/training_time_' +str(x) + '.txt'
                fout = open(fileName, 'a')
                fout.write('After Test:\n')
                for i in xrange(len(result)):
                    fout.write(str(result[i][0]))
                    fout.write('\n')
                fout.close()
                # end for test
        time.sleep(0.2)
示例#4
0
def main():

    # inintialize the database for storing the alarms
    myDB = MongoDBInterface('grande', 27017)
    myDB.SetDB('alarm_filter')
    myDB.SetCollection('photos')

    adi = AlarmDataInterface()

    regions = read_regions()

    for region in regions:
        to_save = {
            'mid_lat': region[0],
            'mid_lng': region[1],
            'valid': False,
            'real_count': -1,
            'mu': -1,
            'std': -1,
            'photos': [],
            'zscore': -1
        }
        photos = []
        try:
            ts, photos = find_photos_given_region(region[0], region[1],
                                                  '15Min', 'tmp_citybeat',
                                                  True)
        except Exception as e:
            save_to_mongo(to_save)
            continue
        sz = len(ts)
        cur_value = 0
        if sz > 0:
            cur_time = ts.index[len(ts.index) - 1]
            if isnull(ts.values[len(ts.values) - 1]):
                cur_value = -1
            else:
                cur_value = ts.values[len(ts.values) - 1]
        else:
            save_to_mongo(to_save)
            continue
        if total_seconds(datetime.utcnow() - cur_time) > 900:
            # there's no latest photo at this region
            save_to_mongo(to_save)
            continue

        try:
            predictions = get_prediction(str(region[0]), str(region[1]))
        except Exception as e:
            save_to_mongo(to_save)
            continue
        predict = -1
        for p in predictions:
            seconds = total_seconds(p['time'] - cur_time)
            if (seconds <= 3600 and seconds >= 0):
                #print 'p time is ',p['time'], 'cur_time is ',cur_time
                predict = p
        if predict == -1:
            save_to_mongo(to_save)
            continue

        mu = float(predict['mu']) / 4.0
        std = sqrt(float(predict['var'])) / 4.0
        zscore = (cur_value - mu) / std

        within_range = float(predict['mu']) / 4.0 + 3 * sqrt(
            float(predict['var'])) / 4.0

        photos_to_save = []
        for photo in photos:
            delta = total_seconds(
                cur_time -
                datetime.utcfromtimestamp(float(photo['created_time'])))
            if delta >= 0 and delta <= 900:
                photo['label'] = 'unlabeled'
                photos_to_save.append(photo)

        to_save['photos'] = photos_to_save
        to_save['real_count'] = cur_value
        to_save['valid'] = True
        to_save['std'] = sqrt(float(predict['var'])) / 4.0
        to_save['mu'] = float(predict['mu']) / 4.0
        to_save['zscore'] = zscore
        save_to_mongo(to_save)

        if zscore >= 2.0 and cur_value >= 3:
            print datetime.utcnow()
            print region[0], ",", region[1]
            print float(predict['mu']) / 4.0, sqrt(
                float(predict['var']
                      )) / 4.0, 'range ', within_range, 'real-> ', cur_value
            for photo in photos_to_save:
                print 'printing photos: ', photo['link'], photo[
                    'created_time'], photo['id'], datetime.utcfromtimestamp(
                        float(photo['created_time']))

            #group photos into an unlabeled event and save it to the db
            predicted_mu = float(predict['mu']) / 4.0
            predicted_std = sqrt(float(predict['var'])) / 4.0
            newEvent = {
                'created_time': datetime.utcnow(),
                'mid_lat': region[0],
                'mid_lng': region[1],
                'predicted_mu': predicted_mu,
                'predicted_std': predicted_std,
                'actual_value': cur_value,
                'zscore': zscore,
                'photos': photos_to_save,
                'label': 'unlabeled'
            }

            if not adi.MergeEvent(newEvent):
                print 'created an event'
                myDB.SaveItem(newEvent)
            else:
                print 'merged an event'
            print '\n'
示例#5
0
def main():
	
	  # inintialize the database for storing the alarms
    myDB = MongoDBInterface('grande', 27017)
    myDB.SetDB('alarm_filter')
    myDB.SetCollection('photos')
    
    regions = read_regions()
    

    for region in regions:
        to_save = {'mid_lat':region[0], 'mid_lng':region[1], 'valid':False, 'real_count':-1,'mu':-1, 'std':-1, 'photos':[], 'zscore':-1};
        photos = []
        try:
            ts, photos = find_photos_given_region(region[0], region[1], '15Min', 'tmp_citybeat', True)
        except Exception as e:
            save_to_mongo(to_save)
            continue
        sz = len(ts)
        cur_value = 0
        if sz>0:
            cur_time = ts.index[len(ts.index)-1]
            if isnull(ts.values[len(ts.values)-1]):
                cur_value = -1
            else:
                cur_value = ts.values[len(ts.values)-1]
        else:
            save_to_mongo(to_save)
            continue
        if total_seconds(datetime.utcnow() - cur_time)>900:
            # there's no latest photo at this region
            save_to_mongo(to_save)
            continue

        try:
            predictions =  get_prediction(str(region[0]), str(region[1]))
        except Exception as e:
            save_to_mongo(to_save)
            continue
        predict = -1
        for p in predictions:
            seconds = total_seconds( p['time']-cur_time)
            if( seconds<=3600 and seconds>=0):
                #print 'p time is ',p['time'], 'cur_time is ',cur_time
                predict = p
        if predict==-1:
            save_to_mongo(to_save)
            continue
        
        mu = float(predict['mu'])/4.0
        std = sqrt(float(predict['var']))/4.0
        zscore = (cur_value - mu)/std

        within_range = float(predict['mu'])/4.0+3*sqrt(float(predict['var']))/4.0
        
        photos_to_save = []
        for photo in photos:
            delta = total_seconds( cur_time - datetime.utcfromtimestamp( float(photo['created_time'])) )
            if delta>=0 and delta<=900:
                photos_to_save.append(photo)

        to_save['photos'] = photos_to_save
        to_save['real_count'] = cur_value
        to_save['valid'] = True
        to_save['std'] = sqrt(float(predict['var']))/4.0
        to_save['mu'] = float(predict['mu'])/4.0
        to_save['zscore'] = zscore
        save_to_mongo(to_save) 
        

        if zscore>=3 and cur_value>=8:
            print datetime.utcnow()
            print region[0],",",region[1]
            print float(predict['mu'])/4.0, sqrt(float(predict['var']))/4.0, 'range ',within_range,'real-> ',cur_value
            for photo in photos_to_save:
                print 'printing photos: ', photo['link'], photo['created_time'], photo['id'], datetime.utcfromtimestamp(float(photo['created_time']))
            print '\n'
            
            #group photos into an unlabeled event and save it to the db  
            predicted_mu = float(predict['mu'])/4.0
            predicted_std = sqrt(float(predict['var']))/4.0
            event = {'discovered_time':datetime.utcnow(), 'lat':region[0], 'lng':region[1], 'predicted_mu':predicted_mu, 'predicted_std':predicted_std, 'actual_value':cur_value, 'zscore':zscore, 'photos':photos_to_save, 'label':'unlabeled'}
            myDB.SaveItem(event)
示例#6
0
def main():
    predict_days = 1
    regions = read_regions()
    redis_conn = Redis('tall4')
    q = Queue(connection=redis_conn)
    cnt = 0
    async_results = {}
    start_time = []
    model_update_time = datetime.utcnow()

    for region in regions:
        par = cnt
        try:
            ts = find_photos_given_region(region[0], region[1])
        except Exception as e:
            print e
            continue
        start, training = process_ts(ts)
        start_time.append(start)
        #testing = get_testing(model_update_time, ( ts.index[len(ts)-1] - start).days, predict_days)
        testing, align = get_testing(model_update_time, start, predict_days)
        print 'start is ', start
        print 'model_update_time is ', model_update_time
        print 'testing is ', testing
        async_results[cnt] = q.enqueue_call(Predict,
                                            args=(
                                                training,
                                                testing,
                                                cnt,
                                            ),
                                            timeout=1720000,
                                            result_ttl=-1)
        # Only for temporal test
        fileName = '/grad/users/kx19/xia/test_tmp/training_time_' + str(
            cnt) + '.txt'
        fout = open(fileName, 'w')
        fout.write('Before Distributed GP:\n')
        for t in testing:
            fout.write(str(t))
            fout.write('\n')
        fout.close()
        # end for test
        cnt += 1
        # Only for temporal test use
        break
        # end for test

    done = False
    begin_time = time.time()
    time.sleep(2)
    saved_flag = [0] * len(async_results)
    while not done:
        print "Time elapsed : ", time.time() - begin_time
        done = True
        for x in range(cnt):
            #print 'checking ',x
            result = async_results[x].return_value
            #print 'check done'
            #print 'res is ',result
            if result is None:
                done = False
                continue
            if saved_flag[x] == 0:
                #result = fix_time(start_time[x], result)
                result = do_align(align, result)
                save_to_mongo(result, regions[x], model_update_time)
                saved_flag[x] = 1
                # Only for temporal test
                fileName = '/grad/users/kx19/xia/test_tmp/training_time_' + str(
                    x) + '.txt'
                fout = open(fileName, 'a')
                fout.write('After Test:\n')
                for i in xrange(len(result)):
                    fout.write(str(result[i][0]))
                    fout.write('\n')
                fout.close()
                # end for test
        time.sleep(0.2)