Example #1
0
def run():
    coordinates = [InstagramConfig.photo_min_lat,
            InstagramConfig.photo_min_lng,
            InstagramConfig.photo_max_lat,
            InstagramConfig.photo_max_lng
                 ]
    huge_region = Region(coordinates)
    
    regions = huge_region.divideRegions(25,25)
    filtered_regions = huge_region.filterRegions( regions )
    regions = filtered_regions

    for r in regions:
        r.display()

    cur_utc_timestamp = getCurrentStampUTC() 
    #experiment start time - Dec 1 00:00
    clock = 1354320000  + 7*24*3600
    end_of_time = 1354320000 + 7*24*3600  + 7*24*3600
    days_passed = 0
    _results =  {} 
    _saved = {}

    redis_conn = Redis("tall4")
    redis_queue = Queue(connection = redis_conn)

    while clock<end_of_time:
        print 'working on day ',days_passed
        days_passed+=1
        # use 14 days of data as training
        fourteen_days_ago = clock - 14*24*3600

        for i in range(len(regions)):
        #for i in range(1):
            test_region = regions[i]
            try:
                gp = GaussianProcessJob( test_region, str(fourteen_days_ago), str(clock) , redis_queue)
                res, pred_time = gp.submit()
            except Exception as e:
                print 'Initialization of gp error. continue, error message %s'%(e)
                continue
            _results[gp.getID()] = (test_region, res, pred_time)
            _saved[ gp.getID() ] = False
        save_to_mongo(_results, _saved, cur_utc_timestamp) 
        clock+=3600*24
    done = False
    while not done:
        done = save_to_mongo(_results, _saved, cur_utc_timestamp)
        time.sleep(10)

    print 'finish work' 
Example #2
0
def run(data_source):
    coordinates = [
        InstagramConfig.photo_min_lat,
        InstagramConfig.photo_min_lng,
        InstagramConfig.photo_max_lat,
        InstagramConfig.photo_max_lng,
    ]
    nyc_region = Region(coordinates)
    regions = nyc_region.divideRegions(25, 25)
    if data_source == "twitter":
        regions = nyc_region.filterRegions(regions, test=True, n=25, m=25, element_type="tweets")
    elif data_source == "instagram":
        regions = nyc_region.filterRegions(regions, test=True, n=25, m=25, element_type="photos")

    for r in regions:
        r.display()

    cur_utc_timestamp = getCurrentStampUTC()

    _results = {}
    _saved = {}

    redis_conn = Redis("tall4")
    redis_queue = Queue(connection=redis_conn)
    fourteen_days_ago = cur_utc_timestamp - 24 * 14 * 3600

    for i in range(len(regions)):
        logging.warn("Working on region %d" % i)
        test_region = regions[i]
        # try:
        gp = GaussianProcessJob(test_region, str(fourteen_days_ago), str(cur_utc_timestamp), redis_queue)
        res, pred_time = gp.submit()
        # except Exception as e:
        #    logging.warn("Initialization of gp error. continue, error message %s" % e)
        #    continue
        _results[gp.getID()] = (test_region, res, pred_time)
        _saved[gp.getID()] = False

    save_to_mongo(_results, _saved, cur_utc_timestamp, data_source)
    done = False
    while not done:
        done = save_to_mongo(_results, _saved, cur_utc_timestamp, data_source)
        time.sleep(10)
        logging.warn("Waiting for completing...")

    logging.warn("Work done.")
Example #3
0
def run():
    coordinates = [InstagramConfig.photo_min_lat,
            InstagramConfig.photo_min_lng,
            InstagramConfig.photo_max_lat,
            InstagramConfig.photo_max_lng
                 ]
    huge_region = Region(coordinates)
    
    regions = huge_region.divideRegions(25,25)
    filtered_regions = huge_region.filterRegions( regions )
    regions = filtered_regions

    for r in regions:
        r.display()

    cur_utc_timestamp = getCurrentStampUTC() 
    
    _results =  {} 
    _saved = {}

    redis_conn = Redis("tall4")
    redis_queue = Queue(connection = redis_conn)
    fourteen_days_ago = cur_utc_timestamp - 24*14*3600

    for i in range(len(regions)):
        test_region = regions[i]
        try:
            gp = GaussianProcessJob( test_region, str(fourteen_days_ago), str(cur_utc_timestamp) , redis_queue)
            res, pred_time = gp.submit()
        except Exception as e:
            print 'Initialization of gp error. continue, error message %s'%(e)
            continue
        _results[gp.getID()] = (test_region, res, pred_time)
        _saved[ gp.getID() ] = False

    save_to_mongo(_results, _saved, cur_utc_timestamp) 
    done = False
    while not done:
        done = save_to_mongo(_results, _saved, cur_utc_timestamp)
        time.sleep(10)

    print 'finish work'