def _compose_tiff(new, path):
master_path = path + "/"+new + ".tif"
# "in _compose_tiff, master is " + master_path
myFiles = []
for filename in sorted(os.listdir(path)):
file_path = os.path.join(path, filename)
uncompressed_file = _uncompress(file_path)
myFiles.append(uncompressed_file)
# print "decompressed file is " + uncompressed_file + " for compressed file " + file_path
mergeCmd = '/usr/local/bin/gdal_merge.py -n -9999 -a_nodata -9999 -of GTIFF -o '
mergeCmd += master_path
for filename in myFiles:
mergeCmd += ' ' + filename
log("Merging..." + mergeCmd)
# print "Merging..." + mergeCmd
p = Popen(mergeCmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
p.wait()
output = p.stdout.read()
assert p.returncode == 0, "GDalMerge Failed"
log(output)
return master_path
def get_cities(location, num_cities=10):
log("Getting new cities from database")
select_query = pgConn.createSelectQuery(location, num_cities)
records = pgConn.performSelect(select_query)
log("Fetched", len(records), location, "records")
return records
def init():
global pgConn
servname = socket.gethostname()
greencitieslog.start()
log("Starting Green Cities on", servname)
log("Connecting to database")
pgConn = dbObj.pgConnection()
def _uncompress(file_path):
cmd = '/usr/local/bin/gdal_translate %s %s.uncompressed.tif' % (file_path, file_path)
log("Calling:",cmd)
p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
p.wait()
out,err = p.communicate()
log("Gdal_transulate", out, err)
assert p.returncode == 0, "Uncompress Failed"
return '%s.uncompressed.tif' % file_path
def writeImg(self):
file_name = self.file_dir + "/" + self.imgname
log("Attemping to write png " + file_name)
try:
wt = png.Writer(width=self.px_w, height=self.px_h, alpha=True, bitdepth=8)
f = open(file_name, 'wb')
wt.write(f, self.rgbs)
f.close()
log("Complete!")
except IOError as e:
print "Error:", str(e)
raise AssertionError(str(e))
def reproject_shapefile(full_shapefile, shapefile, new_projcode, shapefile_tmpDir):
os.mkdir(shapefile_tmpDir)
new_fil = shapefile_tmpDir + "/"+ shapefile
base,ext = shapefile.split(".")
cmd = 'ogr2ogr -s_srs "EPSG:4326" -t_srs "EPSG:' + new_projcode + '" ' + new_fil + " " + full_shapefile
log("Running:", cmd)
# print "Running: " + cmd
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
p.wait()
out, err = p.communicate()
log("ogr2ogr said:", out, err)
assert p.returncode == 0, "Reprojection failed - Probably a SRS missmatch..."
return new_fil
def diagnoseBadSpec(identifierAsList, rawSpec):
log("bad specifier: " + rawSpec)
if len(identifierAsList) != 4:
log("length of derived list must be 4, not " + len(identifierAsList))
return
geomSpecOK = re.match("p[0-9][0-9][0-9]r[0-9][0-9][0-9]", identifierAsList[0])
dateSpecOK = re.match("[5-7]dt[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]", identifierAsList[1])
bandSpecOK = re.match("b[0-9][0-9]", identifierAsList[3])
if not geomSpecOK:
log("bad geometry specifier " + identifierAsList[0])
if not dateSpecOK:
log("bad date specifier " + identifierAsList[1])
if not bandSpecOK:
log("bad band specifier " + identifierAsList[3])
def readImgData(self):
file_name = self.fname
log("Attempting to read image data from " + file_name)
try:
f = open(file_name, 'rb')
imgData = f.read()
image = png.Reader(bytes=imgData)
f.close()
self.px_w, self.px_h, self.rgbs, self.metadata = image.asRGBA()
self.rgbs = list(self.rgbs)
log("Complete!")
except IOError as e:
print "Error:", str(e)
raise AssertionError(str(e)) # crappy but this is all i catch atm
def getImgList(self,pgConn):
""" """
import json
sql = "select fname, ST_AsGeoJson(tiff4326.the_geom) "\
"from (select ST_Transform(the_geom, 4326) as the_geom from map where gid = "+str(self.gid)+") as map , tiff4326 where ST_Intersects(map.the_geom, tiff4326.the_geom);"
records = pgConn.performSelect(sql)
if len(records)==0:
raise MissingCoverage("Could not find any images in the database for "+str(self.gid))
decided_date = None
self.images = []
images = []
for r in records:
is_bad, bb = self.check_bad_record(json.loads(r[1]))
if is_bad:
log("Warning: Skipping " + r[0] + " because of wraparound bug. Coords: " + str(bb))
continue
else:
log("Info: Checking " + r[0])
pieces = r[0].split('_')
if len(pieces) == 4:
# we only want 5dt or 7dt. Landsat 7 (7),decade set (d) and triband (t)
if 'dt' in pieces[1] and checkValidImageSpec(pieces):
images.append(imageID(pieces))
else:
diagnoseBadSpec(pieces, r[0])
for image in images:
subsumed = reduce(lambda x,y: x or y, [otherImage.subsumes(image) for otherImage in images])
if subsumed: continue
self.images.append(image)
log("total images %s total subsumed images %s"%(len(images), len(self.images)))
for image in self.images:
self.files.append(image.fileName)
self.buckets.append((image.bucket, image.fileName))
log( "%s images intersect the city %s" % (len(self.files),self.city))
log( "At the end of getImgList, files are " + " ".join(self.files))
return len(self.files)
def getMostRecentSet(fnames):
""" """
dates = []
for f in fnames:
# we are always encounter 'dt' before the date
fdate = f.split('t')[1].split('.')[0]
if fdate not in dates:
dates.append(int(fdate))
maxdate = str(max(dates))
newfiles = []
for f in fnames:
if maxdate in f:
newfiles.append(f)
log(" in getMostRecentSet, newfiles are " + " ".join(newfiles))
return newfiles
def getSwiftImgs(self):
""" pull required buckets from swift into local dir """
assert(len(self.files))
assert(len(self.buckets))
cwd = os.getcwd()
log("In getSwiftImgs, current directory is " + cwd)
havebucket = False
local_files = []
for b, f in self.buckets:
local_files.append(self.file_manager.get_file(b, f))
self.files = local_files
log(" At the end of getSwiftImgs, files are " + " ".join(self.files))
self.havefiles = True
def combine_bands(allfiles, file_manager, gid="new"):
target = []
dirName = gcswift.get_dir_name(allfiles[0])
assert dirName == file_manager.tmp_file_dir, "band directory " + dirName + " ought to be equal to " + file_manager.tmp_file_dir
if gid != "new" and os.path.exists(dirName + "/" + gid+"_b03.tif"):
prefix = dirName + "/" + gid
return [prefix + "_b03.tif", prefix + "_b04.tif", prefix + "_b07.tif"]
assert dirName == file_manager.tmp_file_dir, "band directory " + dirName + " ought to be equal to " + file_manager.tmp_file_dir
for full_name in allfiles:
fname = gcswift.get_raw_file_name(full_name)
info = grab_pathrow_band_time(fname)
if info[3] != '7dt' and info[3] != '5dt':
log("Skipping non-landsat 5/7 image" + fname)
else:
target.append((fname, info))
cwd = os.getcwd()
os.chdir(dirName)
# all operations now are local to dirName
temps = {'b03':tempfile.mkdtemp(prefix='landsatb03', dir=file_manager.tmp_file_dir),
'b04':tempfile.mkdtemp(prefix='landsatb04', dir=file_manager.tmp_file_dir),
'b07':tempfile.mkdtemp(prefix='landsatb07', dir=file_manager.tmp_file_dir)}
for f in target:
shutil.copy(f[0], temps[f[1][1]]+"/"+f[0])
done = []
for d in temps.items():
new = _compose_tiff(gid+d[0], d[1])
done.append(new.split('/')[-1])
shutil.copy(new,".")
shutil.rmtree(d[1])
# return done
os.chdir(cwd)
# Return the absolute paths
result = []
for item in done:
result.append(dirName + "/" + item)
return result
def uploadToSwift(self):
log("Uploading processed image " + self.img.imgname + " to swift")
cwd = os.getcwd()
os.chdir(self.file_manager.tmp_file_dir)
# print "Current working directory is " + os.getcwd()
p = gcswift.do_swift_command(settings.SWIFT_PROXY2, "upload", settings.SWIFT_PNG_BUCKET, False, self.img.imgname)
log("Swift finished with ", p.returncode)
assert p.returncode == 0, "Failed with %s"%(p.communicate()[1])
log("Complete!")
os.chdir(cwd)
def checkImages(self):
if len(self.images) == 0:
log("No images for " + self.city)
return
if len(self.images) <= 3: return
for image in self.images:
hasNeighbor = reduce(lambda x, y: x or y, [otherImage.overlaps(image) for otherImage in self.images])
if hasNeighbor: continue
log("Isolated image for " + self.city)
for myImage in self.images: log("(%d, %d)" % (myImage.path, myImage.row))
return
def decompressTiffs(files):
""" """
import zlib
tiffs = []
cwd = os.getcwd()
log("In decompressTiffs, cwd is " + cwd)
log("Decompressing files")
try:
for f in files:
# tname = f.rstrip('.gz')
tname = self.file_manager.get_tiff_name(f)
tiffs.append(tname)
# print "In decompress_tiffs: file is " + f + " tname is " + tname
if not os.path.exists(tname):
# inf = gzip.GzipFile(os.path.join(cwd, f), 'rb')
inf = gzip.GzipFile(f, 'rb')
dcmpdata = inf.read()
inf.close()
outf = open(tname, "w") # try catches????
outf.write(dcmpdata)
outf.close()
except IOError as e:
# might be corrupt - delete now so not cached
for f in files:
print "Corrupt files, unlinking:", f
os.unlink(f)
raise AssertionError(e)
except zlib.error as e:
# might be corrupt - delete now so not cached
for f in files:
print "Corrupt files, unlinking:", f
os.unlink(f)
raise AssertionError(e)
log(" At the end of decompressTiffs, files are: " + " ".join(files) + ". Tiffs are:" + " ".join(tiffs))
log("Decompressing files done")
return tiffs
def main(location):
global pgConn
records = get_cities(location)
os.chdir(settings.TEMP_FILE_DIR)
for record in records:
try:
log("Processing", record[GID], record[CITY_NAME])
if record[GID] is None:
log("Null GID, skipping")
continue
process_city(record[GID], record[CITY_NAME], record[CV_HULL], record[XMIN:], location)
except AssertionError as e:
log(e)
#except Exception as e:
# log("Processing ", record[CITY_NAME], " with error ", str(e))
del pgConn
return len(records)
def getSwiftImgs(self):
""" pull required buckets from swift into local dir """
assert(len(self.files))
assert(len(self.buckets))
cwd = os.getcwd()
log("In getSwiftImgs, current directory is " + cwd)
havebucket = False
local_files = []
for b, f in self.buckets:
## if os.path.exists(f):
## # Skipping file "+f+" as we already have it!
## continue
## gcswift.swift("download", b, f)
local_files.append(self.file_manager.get_file(b, f))
self.files = local_files
log(" At the end of getSwiftImgs, files are " + " ".join(self.files))
log("Complete!")
self.havefiles = True
def process_city(gid, cityname, convex_hull, xmin_box, location, testing=False):
assert (gid is not None and convex_hull is not None), "Process city inputs are None"
if cityname == None:
cityname = "No Name with ID:" + str(gid)
greencitieslog.prefix = cityname
# box is floats: [xmin, ymin, xmax, ymax]
box = xmin_box
#img_w, img_h = get_img_size(pgConn, box[0], box[1], box[2], box[3])
#print " -> w:h = ", img_w, ":", img_h
greenspace = 0
#if img_w and img_h:
coord_sys = "EPSG:" + dbObj.GEOG
start_t = datetime.datetime.now()
imgname = "missing"
greenspace = 0.0
servername = socket.gethostname()
if not testing:
lsimg = landsatImg.GrassLandsat(gid, cityname, box, coord_sys, location)
log("Getting Image List")
lsimg.getImgList(pgConn)
log("Getting Images from Swift")
lsimg.getSwiftImgs()
log("Combine into PNG")
lsimg.combineIntoPng()
log("Read Image Data")
lsimg.img.readImgData()
# get polygon as list of points (ie, not a string)
log("Extracting Polygon points")
polygon = wkt_to_list(convex_hull)
log("Calculating Greenspace")
greenspace = calc_greenspace(lsimg.img, polygon)
log("Greenvalue of " ,str(greenspace), " was found")
log("RESULT:",gid, cityname, greenspace)
# pgConn.createUpdateStmnt(greenspace,
# lsimg.gid, lsimg.city,
# start_t, end_t,
# lsimg.img.imgname,
# servname, location)
lsimg.uploadToSwift()
imgname = lsimg.img.imgname
# only in for test
tmp_directory = lsimg.file_manager.tmp_file_dir
del lsimg
if os.path.exists(tmp_directory):
print 'Cleanup failed to delete ' + tmp_directory
else:
print "warning testing mode"
end_t = datetime.datetime.now()
result_dict = {'id':gid,
'name':cityname,
'greenspace_val':greenspace,
'stime':start_t.isoformat(),
'etime':end_t.isoformat(),
'imgname':imgname,
'servername':servername,
'location':location
}
return result_dict
def combineIntoPng(self):
""" """
def getMostRecentSet(fnames):
""" """
dates = []
for f in fnames:
# we are always encounter 'dt' before the date
fdate = f.split('t')[1].split('.')[0]
if fdate not in dates:
dates.append(int(fdate))
maxdate = str(max(dates))
newfiles = []
for f in fnames:
if maxdate in f:
newfiles.append(f)
log(" in getMostRecentSet, newfiles are " + " ".join(newfiles))
return newfiles
def decompressTiffs(files):
""" """
import zlib
tiffs = []
cwd = os.getcwd()
log("In decompressTiffs, cwd is " + cwd)
log("Decompressing files")
try:
for f in files:
# tname = f.rstrip('.gz')
tname = self.file_manager.get_tiff_name(f)
tiffs.append(tname)
# print "In decompress_tiffs: file is " + f + " tname is " + tname
if not os.path.exists(tname):
# inf = gzip.GzipFile(os.path.join(cwd, f), 'rb')
inf = gzip.GzipFile(f, 'rb')
dcmpdata = inf.read()
inf.close()
outf = open(tname, "w") # try catches????
outf.write(dcmpdata)
outf.close()
except IOError as e:
# might be corrupt - delete now so not cached
for f in files:
print "Corrupt files, unlinking:", f
os.unlink(f)
raise AssertionError(e)
except zlib.error as e:
# might be corrupt - delete now so not cached
for f in files:
print "Corrupt files, unlinking:", f
os.unlink(f)
raise AssertionError(e)
log(" At the end of decompressTiffs, files are: " + " ".join(files) + ". Tiffs are:" + " ".join(tiffs))
log("Decompressing files done")
return tiffs
assert(self.havefiles)
assert(len(self.files))
# get raster 3 band images for each prefix
fnames = decompressTiffs(self.files)
#print fnames
if len(fnames) > 3:
log("Combining images")
fnames = combine.combine_bands(fnames, self.file_manager, str(self.gid))
log("Getting shapefile for", str(self.gid))
shpname = trim.getShapefile(self.file_manager.tmp_file_dir, self.gid)
log("Shapefile is " + shpname)
#try:
trim.crop(shpname, fnames[0], fnames[1], fnames[2], prefix="trim_", shapefile_tmpDir = self.file_manager.shapefile_tmpDir)
dir_name = gcswift.get_dir_name(fnames[0])
fnames = [dir_name + "/trim_"+gcswift.get_raw_file_name(name) for name in fnames]
log("New fnames are: " + " ".join(fnames))
#except AssertionError as e:
# print e
namesAndBands = [(fname, getBand(fname)) for fname in fnames]
namesAndBands = sorted(namesAndBands, key=itemgetter(1), reverse=True)
fnames = [fname for (fname, band) in namesAndBands]
pre = self.file_manager.tmp_file_dir + "/" + str(self.gid)
allbandsTIF = pre +"_allbands.tif"
allbandsPNG = pre +"_allbands.png"
mergeCmd = '/usr/local/bin/gdal_merge.py -n -9999 -a_nodata -9999 -separate -o '
mergeCmd += allbandsTIF
for filename in fnames: mergeCmd += ' ' + filename
log("creating... " + allbandsTIF + " with "+ mergeCmd)
p = Popen(mergeCmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
p.wait()
output = p.stdout.read()
assert p.returncode == 0, "Creation of merged bands Failed"
log(output)
#
# Now convert to PNG
#
translateCmd = "/usr/local/bin/gdal_translate -of png -ot Byte -scale %s %s" % (allbandsTIF, allbandsPNG)
log("creating... " + allbandsPNG + " with " + translateCmd)
p = Popen(translateCmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
p.wait()
output = p.stdout.read()
assert p.returncode == 0, "Creation of png Failed"
log(output)
# we give it 1) the name of the png created from grass
# and 2) the name the greenspace calc will create
# and 3) the directory where they will be stored
pngimg = pngImg(allbandsPNG, str(self.gid) + ".png", self.file_manager.tmp_file_dir)
pngimg.bbox = self.bbox
self.img = pngimg
def getImgList(self,pgConn):
""" """
import json
sql = "select fname, ST_AsGeoJson(tiff4326.the_geom) "\
"from (select ST_Transform(the_geom, 4326) as the_geom from map where gid = "+str(self.gid)+") as map , tiff4326 where ST_Intersects(map.the_geom, tiff4326.the_geom);"
records = pgConn.performSelect(sql)
if len(records)==0:
raise MissingCoverage("Could not find any images in the database for "+str(self.gid))
decided_date = None
self.images = []
images = []
for r in records:
is_bad, bb = self.check_bad_record(json.loads(r[1]))
if is_bad:
log("Warning: Skipping " + r[0] + " because of wraparound bug. Coords: " + str(bb))
continue
else:
log("Info: Proccessing " + r[0])
pieces = r[0].split('_')
if len(pieces) == 4:
# we only want 5dt or 7dt for landsat 7 (7),decade set (d) and triband (t)
if 'dt' in pieces[1] and checkValidImageSpec(pieces):
images.append(imageID(pieces))
else:
diagnoseBadSpec(pieces, r[0])
# images1 = []
for image in images:
subsumed = reduce(lambda x,y: x or y, [otherImage.subsumes(image) for otherImage in images])
if subsumed: continue
self.images.append(image)
# images1.append(image)
# Make sure each image has a neighbor
## for image in images1:
## hasNeighbor = reduce(lambda x, y: x or y, [otherImage.overlaps(image) for otherImage in images1])
## if hasNeighbor: self.images.append(image)
## # If there are no connected images, just pick the one with the least path, least
## # row; any choice will do, we just need it to be consistent across bands
## if len(self.images) == 0:
## images1 = sorted(images1, key=attrgetter('path', 'row'))
## self.images = [images1[0]]
# print "total images", len(images), "total unsubsumed images", len(images1), "total connected images", len(self.images)
log("total images %s total subsumed images %s"%(len(images), len(self.images)))
for image in self.images:
self.files.append(image.fileName)
self.buckets.append((image.bucket, image.fileName))
log( "%s images intersect the city %s" % (len(self.files),self.city))
log( "At the end of getImgs, files are " + " ".join(self.files))
return len(self.files)
