def populate_cities(ncities, testing_prefix="", testing=False):
# get command line options
print "Running with "+str(ncities)+" cities to process"
here = os.path.dirname(os.path.realpath(__file__))+"/"
cities = []
greenspace.init()
# we break the world into 4 'regions' so we select ncities from each region
#for region_num in xrange(1,5):
region_num = 1
cities.extend(greenspace.get_cities(region_num, ncities))
if testing:
cities = []
for i in xrange(0,ncities):
cities.append([FAKE_CITY])
print cities
manager = taskmanager.TaskManager(prefix=testing_prefix)
manager.reset()
for i,city in enumerate(cities):
job = json.dumps(city ,separators=(',',':'))
if testing:
manager.add_task({'task':'greencities','data':job})
else:
clust = decide_cluster(i, ncities)
print ">> Enqueue", city[1], "on", taskmanager._sites[clust]
manager.add_task({'task':'greencities','data':job}, clust )
return manager.get_size()
def process_results(prefix="", testing=False, blocking=True):
submitted = []
error_log = open("/tmp/error.log", 'w')
fails = 0
passes = 0
try:
client = taskmanager.TaskClient(queue=prefix+taskmanager.RESULT_QUEUE_NAME)
if not testing:
greenspace.init()
while(True):
try:
print "Getting new task:"
if blocking:
new_job, jobid = client.blocking_get_task()
else:
new_job, jobid = client.get_task()
if new_job == None:
return submitted
if 'result' in new_job and new_job['result'] != u'failure':
job = json.loads(new_job['result'])
print job['name'], "worked with", job['greenspace_val'], "greenspace in", job['imgname']
if not testing:
submit_result(**job)
passes += 1
submitted.append(job)
else:
result = new_job.get('result','Failed')
name = new_job.get('name', "Missing Name")
err_msg = new_job.get('message', "There was no message passed back.")
message = str(result) + " on " + str(name) + ", failed with: " + str(err_msg) + "\n"
print message,
fails += 1
error_log.write(message)
submitted.append(new_job)
client.report_done(jobid)
except ValueError as e:
print traceback.format_exc()
print type(e), str(e), "New Job:",type(new_job), new_job
except Exception as e:
print type(e), str(e),traceback.format_exc()
if testing:
raise e
finally:
print fails,"jobs failed of", passes+fails, "total."
greenspace.close()
error_log.close()
def process_cities(testing_prefix="", testing=False):
client = taskmanager.TaskClient(prefix=testing_prefix)
print "Processing:", client.queue, "queue."
ndone = 0
while(True):
results = client.get_task()
if results == None or results == (None,None):
return ndone
new_job, jobid = results
os.chdir(settings.TEMP_FILE_DIR)
greencitieslog.start()
print new_job, jobid
try:
if not testing:
greenspace.init()
green_results = greenspace.process_city_from_json(new_job['data'], testing=testing)
else:
green_results = mq_calc.FAKE_RESULT
except IOError as e:
# no matter what, on ioerror die, we are probably out of space.
print str(e)
sys.exit(1)
raise e
except gcswift.MissingSwiftFile as e:
client.report_done(jobid, {'task':'greencity', 'name':"fixme", 'result':'failure','message':e.message.translate(None,"\n\\/'")})
except landsatImg.MissingCoverage as e:
client.report_done(jobid, {'task':'greencity', 'name':"fixme", 'result':'failure','message':e.message.translate(None,"\n\\/'")})
except Exception as e:
if settings.PRODUCTION_MODE:
client.report_done(jobid, {'task':'greencity', 'name':"fixme", 'result':'failure','message':str(e).translate(None,"\n\\/'")})
else:
raise e
else:
client.report_done(jobid, {'task':'greencity', 'result':json.dumps(green_results)})
ndone += 1
greencitieslog.close()
def process_cities(testing_prefix="", testing=False):
client = taskmanager.TaskClient(prefix=testing_prefix)
print "Processing:", client.queue, "queue."
ndone = 0
while(True):
new_job, jobid = client.get_task()
if new_job == None:
return ndone
os.chdir(settings.TEMP_FILE_DIR)
greencitieslog.start()
id, name, poly, bb1, bb2, bb3, bb4 = json.loads(new_job['data'])
try:
if not testing:
greenspace.init()
location = taskmanager.get_local_site_name()
green_results = greenspace.process_city(id,name,poly,(bb1,bb2,bb3,bb4),location, testing=testing)
else:
green_results = mq_calc.FAKE_RESULT
except IOError as e:
# no matter what, on ioerror die, we are probably out of space.
sys.exit(1)
raise e
except gcswift.MissingSwiftFile as e:
client.report_done(jobid, {'task':'greencity', 'name':name, 'result':'failure','message':e.message.translate(None,"\n\\/'")})
except landsatImg.MissingCoverage as e:
client.report_done(jobid, {'task':'greencity', 'name':name, 'result':'failure','message':e.message.translate(None,"\n\\/'")})
except Exception as e:
if settings.PRODUCTION_MODE:
client.report_done(jobid, {'task':'greencity', 'name':name, 'result':'failure','message':e.message.translate(None,"\n\\/'")})
else:
raise e
else:
client.report_done(jobid, {'task':'greencity', 'result':json.dumps(green_results)})
ndone += 1
greencitieslog.close()
def mapper_init(x,y):
print "Mapper Init"
greenspace.init()
print "Mapper Init Done"
def mapper_init(x,y):
print "Mapper Init"
greenspace.init()
print "Mapper Init Done"
if __name__ == '__main__':
parser = optparse.OptionParser()
parser.add_option("-c", "--num_cities", dest="num_cities", type="int", default=2, help="number of cities to run the calculation on")
parser.add_option("-m", "--num_mappers", dest="num_mappers", type="int", default=4, help="number of nodes to map the job to")
(options, args_not_used) = parser.parse_args()
print "Running with "+str(options.num_cities)+" cities and "+str(options.num_mappers)+" mappers"
here = os.path.dirname(os.path.realpath(__file__))+"/"
greenspace.init()
cities = greenspace.get_cities(1, options.num_cities)
input_files = []
fds = []
for n in xrange(options.num_mappers):
fid,name = tempfile.mkstemp(dir=".")
print "Temp data file", fid, name
f = os.fdopen(fid, "w+b")
fds.append(f)
input_files.append(name)
for i,c in enumerate(cities):
fd = fds[i%len(input_files)]
json.dump(c,fd ,separators=(',',':'))
fd.write("\n")
print "City", c[1], "to",fd