def run():
coordinates = [
InstagramConfig.photo_min_lat, InstagramConfig.photo_min_lng,
InstagramConfig.photo_max_lat, InstagramConfig.photo_max_lng
]
huge_region = Region(coordinates)
regions = huge_region.divideRegions(25, 25)
filtered_regions = huge_region.filterRegions(regions)
regions = filtered_regions
for r in regions:
r.display()
cur_utc_timestamp = getCurrentStampUTC()
#experiment start time - Dec 1 00:00
clock = 1354320000 + 7 * 24 * 3600
end_of_time = 1354320000 + 7 * 24 * 3600 + 7 * 24 * 3600
days_passed = 0
_results = {}
_saved = {}
redis_conn = Redis("tall4")
redis_queue = Queue(connection=redis_conn)
while clock < end_of_time:
print 'working on day ', days_passed
days_passed += 1
# use 14 days of data as training
fourteen_days_ago = clock - 14 * 24 * 3600
for i in range(len(regions)):
#for i in range(1):
test_region = regions[i]
try:
gp = GaussianProcessJob(test_region, str(fourteen_days_ago),
str(clock), redis_queue)
res, pred_time = gp.submit()
except Exception as e:
print 'Initialization of gp error. continue, error message %s' % (
e)
continue
_results[gp.getID()] = (test_region, res, pred_time)
_saved[gp.getID()] = False
save_to_mongo(_results, _saved, cur_utc_timestamp)
clock += 3600 * 24
done = False
while not done:
done = save_to_mongo(_results, _saved, cur_utc_timestamp)
time.sleep(10)
print 'finish work'
def run():
coordinates = [InstagramConfig.photo_min_lat,
InstagramConfig.photo_min_lng,
InstagramConfig.photo_max_lat,
InstagramConfig.photo_max_lng
]
huge_region = Region(coordinates)
regions = huge_region.divideRegions(25,25)
filtered_regions = huge_region.filterRegions( regions )
regions = filtered_regions
for r in regions:
r.display()
cur_utc_timestamp = getCurrentStampUTC()
#experiment start time - Dec 1 00:00
clock = 1354320000 + 7*24*3600
end_of_time = 1354320000 + 7*24*3600 + 7*24*3600
days_passed = 0
_results = {}
_saved = {}
redis_conn = Redis("tall4")
redis_queue = Queue(connection = redis_conn)
while clock<end_of_time:
print 'working on day ',days_passed
days_passed+=1
# use 14 days of data as training
fourteen_days_ago = clock - 14*24*3600
for i in range(len(regions)):
#for i in range(1):
test_region = regions[i]
try:
gp = GaussianProcessJob( test_region, str(fourteen_days_ago), str(clock) , redis_queue)
res, pred_time = gp.submit()
except Exception as e:
print 'Initialization of gp error. continue, error message %s'%(e)
continue
_results[gp.getID()] = (test_region, res, pred_time)
_saved[ gp.getID() ] = False
save_to_mongo(_results, _saved, cur_utc_timestamp)
clock+=3600*24
done = False
while not done:
done = save_to_mongo(_results, _saved, cur_utc_timestamp)
time.sleep(10)
print 'finish work'
def run(data_source):
coordinates = [
InstagramConfig.photo_min_lat,
InstagramConfig.photo_min_lng,
InstagramConfig.photo_max_lat,
InstagramConfig.photo_max_lng,
]
nyc_region = Region(coordinates)
regions = nyc_region.divideRegions(25, 25)
if data_source == "twitter":
regions = nyc_region.filterRegions(regions, test=True, n=25, m=25, element_type="tweets")
elif data_source == "instagram":
regions = nyc_region.filterRegions(regions, test=True, n=25, m=25, element_type="photos")
for r in regions:
r.display()
cur_utc_timestamp = getCurrentStampUTC()
_results = {}
_saved = {}
redis_conn = Redis("tall4")
redis_queue = Queue(connection=redis_conn)
fourteen_days_ago = cur_utc_timestamp - 24 * 14 * 3600
for i in range(len(regions)):
logging.warn("Working on region %d" % i)
test_region = regions[i]
# try:
gp = GaussianProcessJob(test_region, str(fourteen_days_ago), str(cur_utc_timestamp), redis_queue)
res, pred_time = gp.submit()
# except Exception as e:
# logging.warn("Initialization of gp error. continue, error message %s" % e)
# continue
_results[gp.getID()] = (test_region, res, pred_time)
_saved[gp.getID()] = False
save_to_mongo(_results, _saved, cur_utc_timestamp, data_source)
done = False
while not done:
done = save_to_mongo(_results, _saved, cur_utc_timestamp, data_source)
time.sleep(10)
logging.warn("Waiting for completing...")
logging.warn("Work done.")
def run():
coordinates = [
InstagramConfig.photo_min_lat, InstagramConfig.photo_min_lng,
InstagramConfig.photo_max_lat, InstagramConfig.photo_max_lng
]
huge_region = Region(coordinates)
regions = huge_region.divideRegions(25, 25)
filtered_regions = huge_region.filterRegions(regions)
regions = filtered_regions
for r in regions:
r.display()
cur_utc_timestamp = getCurrentStampUTC()
_results = {}
_saved = {}
redis_conn = Redis("tall4")
redis_queue = Queue(connection=redis_conn)
fourteen_days_ago = cur_utc_timestamp - 24 * 14 * 3600
for i in range(len(regions)):
test_region = regions[i]
try:
gp = GaussianProcessJob(test_region, str(fourteen_days_ago),
str(cur_utc_timestamp), redis_queue)
res, pred_time = gp.submit()
except Exception as e:
print 'Initialization of gp error. continue, error message %s' % (
e)
continue
_results[gp.getID()] = (test_region, res, pred_time)
_saved[gp.getID()] = False
save_to_mongo(_results, _saved, cur_utc_timestamp)
done = False
while not done:
done = save_to_mongo(_results, _saved, cur_utc_timestamp)
time.sleep(10)
print 'finish work'
def run():
coordinates = [InstagramConfig.photo_min_lat,
InstagramConfig.photo_min_lng,
InstagramConfig.photo_max_lat,
InstagramConfig.photo_max_lng
]
huge_region = Region(coordinates)
regions = huge_region.divideRegions(25,25)
filtered_regions = huge_region.filterRegions( regions )
regions = filtered_regions
for r in regions:
r.display()
cur_utc_timestamp = getCurrentStampUTC()
_results = {}
_saved = {}
redis_conn = Redis("tall4")
redis_queue = Queue(connection = redis_conn)
fourteen_days_ago = cur_utc_timestamp - 24*14*3600
for i in range(len(regions)):
test_region = regions[i]
try:
gp = GaussianProcessJob( test_region, str(fourteen_days_ago), str(cur_utc_timestamp) , redis_queue)
res, pred_time = gp.submit()
except Exception as e:
print 'Initialization of gp error. continue, error message %s'%(e)
continue
_results[gp.getID()] = (test_region, res, pred_time)
_saved[ gp.getID() ] = False
save_to_mongo(_results, _saved, cur_utc_timestamp)
done = False
while not done:
done = save_to_mongo(_results, _saved, cur_utc_timestamp)
time.sleep(10)
print 'finish work'