def cleanup_cache(self, respect_file_whitelist = False):
if taskmanager.get_local_site_name() == 'u-tokyo.ac.jp':
os.unlink(self.directory)
os.mkdir(self.directory)
try:
os.chmod(self.directory, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
except OSError:
print "Failed to change mode of " + self.directory + " to 777"
return
files_by_atime = files_by_access_time(self.directory)
max_size_in_bytes = self.max_size_in_kbytes << 10 # convert from kbytes to bytes
total_size = get_dir_size(self.directory, files_by_atime)
if (total_size <= max_size_in_bytes): return
if respect_file_whitelist:
files_to_comb = []
for (atime, filepath, size) in files_by_atime:
if not (filepath in self.file_whitelist):
files_to_comb.append((atime, filepath, size))
else: files_to_comb = files_by_atime
clear_room_in_dir(self.directory, total_size - max_size_in_bytes, files_to_comb)
def get_file(self, bucket, file_name):
file_path = self.directory + "/" + file_name
if(not os.path.exists(self.directory)):
print "Directory ", self.directory, " does not exist"
return None
if taskmanager.get_local_site_name() == 'u-tokyo.ac.jp':
return self.download_special_case_tokyo(bucket, file_name)
if os.path.exists(self.directory + "/" + file_name):
# set the path's utime so LRU's don't get it
os.utime(file_path, None)
else:
os.chdir(self.directory) # so swift downloads to the right directory
swift("download", bucket, file_name, file_name)
if (not os.path.exists(file_path)):
print "File " + file_name + " was not downloaded from swift from bucket " + bucket
return None
else:
os.chmod(file_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
self.file_whitelist.append(file_path)
return file_path
def process_cities(testing_prefix="", testing=False):
client = taskmanager.TaskClient(prefix=testing_prefix)
print "Processing:", client.queue, "queue."
ndone = 0
while(True):
new_job, jobid = client.get_task()
if new_job == None:
return ndone
os.chdir(settings.TEMP_FILE_DIR)
greencitieslog.start()
id, name, poly, bb1, bb2, bb3, bb4 = json.loads(new_job['data'])
try:
if not testing:
greenspace.init()
location = taskmanager.get_local_site_name()
green_results = greenspace.process_city(id,name,poly,(bb1,bb2,bb3,bb4),location, testing=testing)
else:
green_results = mq_calc.FAKE_RESULT
except IOError as e:
# no matter what, on ioerror die, we are probably out of space.
sys.exit(1)
raise e
except gcswift.MissingSwiftFile as e:
client.report_done(jobid, {'task':'greencity', 'name':name, 'result':'failure','message':e.message.translate(None,"\n\\/'")})
except landsatImg.MissingCoverage as e:
client.report_done(jobid, {'task':'greencity', 'name':name, 'result':'failure','message':e.message.translate(None,"\n\\/'")})
except Exception as e:
if settings.PRODUCTION_MODE:
client.report_done(jobid, {'task':'greencity', 'name':name, 'result':'failure','message':e.message.translate(None,"\n\\/'")})
else:
raise e
else:
client.report_done(jobid, {'task':'greencity', 'result':json.dumps(green_results)})
ndone += 1
greencitieslog.close()
def process_city_from_json(json_string, testing=False):
id, name, poly, bb1, bb2, bb3, bb4 = json.loads(json_string)
location = taskmanager.get_local_site_name()
green_results = process_city(id, name, poly, (bb1, bb2, bb3, bb4), location, testing=testing)
return green_results
def get_cluster_swift_proxy():
cluster_name = taskmanager.get_local_site_name()
assert cluster_name in swift_proxies, "%s not in swift_proxies" % cluster_name
return swift_proxies[cluster_name]
def get_cluster_user_id():
cluster_name = taskmanager.get_local_site_name()
assert cluster_name in user_ids, "%s not in user_ids" % cluster_name
return 'system:' + user_ids[cluster_name]
def get_cluster_tmp_location():
cluster_name = taskmanager.get_local_site_name()
assert cluster_name in tmp_dirs, "%s not in tmp_dirs" % cluster_name
return tmp_dirs[cluster_name]
