def _compose_tiff(new, path): master_path = path + "/"+new + ".tif" # "in _compose_tiff, master is " + master_path myFiles = [] for filename in sorted(os.listdir(path)): file_path = os.path.join(path, filename) uncompressed_file = _uncompress(file_path) myFiles.append(uncompressed_file) # print "decompressed file is " + uncompressed_file + " for compressed file " + file_path mergeCmd = '/usr/local/bin/gdal_merge.py -n -9999 -a_nodata -9999 -of GTIFF -o ' mergeCmd += master_path for filename in myFiles: mergeCmd += ' ' + filename log("Merging..." + mergeCmd) # print "Merging..." + mergeCmd p = Popen(mergeCmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) p.wait() output = p.stdout.read() assert p.returncode == 0, "GDalMerge Failed" log(output) return master_path
def get_cities(location, num_cities=10): log("Getting new cities from database") select_query = pgConn.createSelectQuery(location, num_cities) records = pgConn.performSelect(select_query) log("Fetched", len(records), location, "records") return records
def init(): global pgConn servname = socket.gethostname() greencitieslog.start() log("Starting Green Cities on", servname) log("Connecting to database") pgConn = dbObj.pgConnection()
def _uncompress(file_path): cmd = '/usr/local/bin/gdal_translate %s %s.uncompressed.tif' % (file_path, file_path) log("Calling:",cmd) p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) p.wait() out,err = p.communicate() log("Gdal_transulate", out, err) assert p.returncode == 0, "Uncompress Failed" return '%s.uncompressed.tif' % file_path
def writeImg(self): file_name = self.file_dir + "/" + self.imgname log("Attemping to write png " + file_name) try: wt = png.Writer(width=self.px_w, height=self.px_h, alpha=True, bitdepth=8) f = open(file_name, 'wb') wt.write(f, self.rgbs) f.close() log("Complete!") except IOError as e: print "Error:", str(e) raise AssertionError(str(e))
def reproject_shapefile(full_shapefile, shapefile, new_projcode, shapefile_tmpDir): os.mkdir(shapefile_tmpDir) new_fil = shapefile_tmpDir + "/"+ shapefile base,ext = shapefile.split(".") cmd = 'ogr2ogr -s_srs "EPSG:4326" -t_srs "EPSG:' + new_projcode + '" ' + new_fil + " " + full_shapefile log("Running:", cmd) # print "Running: " + cmd p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) p.wait() out, err = p.communicate() log("ogr2ogr said:", out, err) assert p.returncode == 0, "Reprojection failed - Probably a SRS missmatch..." return new_fil
def diagnoseBadSpec(identifierAsList, rawSpec): log("bad specifier: " + rawSpec) if len(identifierAsList) != 4: log("length of derived list must be 4, not " + len(identifierAsList)) return geomSpecOK = re.match("p[0-9][0-9][0-9]r[0-9][0-9][0-9]", identifierAsList[0]) dateSpecOK = re.match("[5-7]dt[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]", identifierAsList[1]) bandSpecOK = re.match("b[0-9][0-9]", identifierAsList[3]) if not geomSpecOK: log("bad geometry specifier " + identifierAsList[0]) if not dateSpecOK: log("bad date specifier " + identifierAsList[1]) if not bandSpecOK: log("bad band specifier " + identifierAsList[3])
def readImgData(self): file_name = self.fname log("Attempting to read image data from " + file_name) try: f = open(file_name, 'rb') imgData = f.read() image = png.Reader(bytes=imgData) f.close() self.px_w, self.px_h, self.rgbs, self.metadata = image.asRGBA() self.rgbs = list(self.rgbs) log("Complete!") except IOError as e: print "Error:", str(e) raise AssertionError(str(e)) # crappy but this is all i catch atm
def getImgList(self,pgConn): """ """ import json sql = "select fname, ST_AsGeoJson(tiff4326.the_geom) "\ "from (select ST_Transform(the_geom, 4326) as the_geom from map where gid = "+str(self.gid)+") as map , tiff4326 where ST_Intersects(map.the_geom, tiff4326.the_geom);" records = pgConn.performSelect(sql) if len(records)==0: raise MissingCoverage("Could not find any images in the database for "+str(self.gid)) decided_date = None self.images = [] images = [] for r in records: is_bad, bb = self.check_bad_record(json.loads(r[1])) if is_bad: log("Warning: Skipping " + r[0] + " because of wraparound bug. Coords: " + str(bb)) continue else: log("Info: Checking " + r[0]) pieces = r[0].split('_') if len(pieces) == 4: # we only want 5dt or 7dt. Landsat 7 (7),decade set (d) and triband (t) if 'dt' in pieces[1] and checkValidImageSpec(pieces): images.append(imageID(pieces)) else: diagnoseBadSpec(pieces, r[0]) for image in images: subsumed = reduce(lambda x,y: x or y, [otherImage.subsumes(image) for otherImage in images]) if subsumed: continue self.images.append(image) log("total images %s total subsumed images %s"%(len(images), len(self.images))) for image in self.images: self.files.append(image.fileName) self.buckets.append((image.bucket, image.fileName)) log( "%s images intersect the city %s" % (len(self.files),self.city)) log( "At the end of getImgList, files are " + " ".join(self.files)) return len(self.files)
def getMostRecentSet(fnames): """ """ dates = [] for f in fnames: # we are always encounter 'dt' before the date fdate = f.split('t')[1].split('.')[0] if fdate not in dates: dates.append(int(fdate)) maxdate = str(max(dates)) newfiles = [] for f in fnames: if maxdate in f: newfiles.append(f) log(" in getMostRecentSet, newfiles are " + " ".join(newfiles)) return newfiles
def getSwiftImgs(self): """ pull required buckets from swift into local dir """ assert(len(self.files)) assert(len(self.buckets)) cwd = os.getcwd() log("In getSwiftImgs, current directory is " + cwd) havebucket = False local_files = [] for b, f in self.buckets: local_files.append(self.file_manager.get_file(b, f)) self.files = local_files log(" At the end of getSwiftImgs, files are " + " ".join(self.files)) self.havefiles = True
def combine_bands(allfiles, file_manager, gid="new"): target = [] dirName = gcswift.get_dir_name(allfiles[0]) assert dirName == file_manager.tmp_file_dir, "band directory " + dirName + " ought to be equal to " + file_manager.tmp_file_dir if gid != "new" and os.path.exists(dirName + "/" + gid+"_b03.tif"): prefix = dirName + "/" + gid return [prefix + "_b03.tif", prefix + "_b04.tif", prefix + "_b07.tif"] assert dirName == file_manager.tmp_file_dir, "band directory " + dirName + " ought to be equal to " + file_manager.tmp_file_dir for full_name in allfiles: fname = gcswift.get_raw_file_name(full_name) info = grab_pathrow_band_time(fname) if info[3] != '7dt' and info[3] != '5dt': log("Skipping non-landsat 5/7 image" + fname) else: target.append((fname, info)) cwd = os.getcwd() os.chdir(dirName) # all operations now are local to dirName temps = {'b03':tempfile.mkdtemp(prefix='landsatb03', dir=file_manager.tmp_file_dir), 'b04':tempfile.mkdtemp(prefix='landsatb04', dir=file_manager.tmp_file_dir), 'b07':tempfile.mkdtemp(prefix='landsatb07', dir=file_manager.tmp_file_dir)} for f in target: shutil.copy(f[0], temps[f[1][1]]+"/"+f[0]) done = [] for d in temps.items(): new = _compose_tiff(gid+d[0], d[1]) done.append(new.split('/')[-1]) shutil.copy(new,".") shutil.rmtree(d[1]) # return done os.chdir(cwd) # Return the absolute paths result = [] for item in done: result.append(dirName + "/" + item) return result
def uploadToSwift(self): log("Uploading processed image " + self.img.imgname + " to swift") cwd = os.getcwd() os.chdir(self.file_manager.tmp_file_dir) # print "Current working directory is " + os.getcwd() p = gcswift.do_swift_command(settings.SWIFT_PROXY2, "upload", settings.SWIFT_PNG_BUCKET, False, self.img.imgname) log("Swift finished with ", p.returncode) assert p.returncode == 0, "Failed with %s"%(p.communicate()[1]) log("Complete!") os.chdir(cwd)
def checkImages(self): if len(self.images) == 0: log("No images for " + self.city) return if len(self.images) <= 3: return for image in self.images: hasNeighbor = reduce(lambda x, y: x or y, [otherImage.overlaps(image) for otherImage in self.images]) if hasNeighbor: continue log("Isolated image for " + self.city) for myImage in self.images: log("(%d, %d)" % (myImage.path, myImage.row)) return
def decompressTiffs(files): """ """ import zlib tiffs = [] cwd = os.getcwd() log("In decompressTiffs, cwd is " + cwd) log("Decompressing files") try: for f in files: # tname = f.rstrip('.gz') tname = self.file_manager.get_tiff_name(f) tiffs.append(tname) # print "In decompress_tiffs: file is " + f + " tname is " + tname if not os.path.exists(tname): # inf = gzip.GzipFile(os.path.join(cwd, f), 'rb') inf = gzip.GzipFile(f, 'rb') dcmpdata = inf.read() inf.close() outf = open(tname, "w") # try catches???? outf.write(dcmpdata) outf.close() except IOError as e: # might be corrupt - delete now so not cached for f in files: print "Corrupt files, unlinking:", f os.unlink(f) raise AssertionError(e) except zlib.error as e: # might be corrupt - delete now so not cached for f in files: print "Corrupt files, unlinking:", f os.unlink(f) raise AssertionError(e) log(" At the end of decompressTiffs, files are: " + " ".join(files) + ". Tiffs are:" + " ".join(tiffs)) log("Decompressing files done") return tiffs
def main(location): global pgConn records = get_cities(location) os.chdir(settings.TEMP_FILE_DIR) for record in records: try: log("Processing", record[GID], record[CITY_NAME]) if record[GID] is None: log("Null GID, skipping") continue process_city(record[GID], record[CITY_NAME], record[CV_HULL], record[XMIN:], location) except AssertionError as e: log(e) #except Exception as e: # log("Processing ", record[CITY_NAME], " with error ", str(e)) del pgConn return len(records)
def getSwiftImgs(self): """ pull required buckets from swift into local dir """ assert(len(self.files)) assert(len(self.buckets)) cwd = os.getcwd() log("In getSwiftImgs, current directory is " + cwd) havebucket = False local_files = [] for b, f in self.buckets: ## if os.path.exists(f): ## # Skipping file "+f+" as we already have it! ## continue ## gcswift.swift("download", b, f) local_files.append(self.file_manager.get_file(b, f)) self.files = local_files log(" At the end of getSwiftImgs, files are " + " ".join(self.files)) log("Complete!") self.havefiles = True
def process_city(gid, cityname, convex_hull, xmin_box, location, testing=False): assert (gid is not None and convex_hull is not None), "Process city inputs are None" if cityname == None: cityname = "No Name with ID:" + str(gid) greencitieslog.prefix = cityname # box is floats: [xmin, ymin, xmax, ymax] box = xmin_box #img_w, img_h = get_img_size(pgConn, box[0], box[1], box[2], box[3]) #print " -> w:h = ", img_w, ":", img_h greenspace = 0 #if img_w and img_h: coord_sys = "EPSG:" + dbObj.GEOG start_t = datetime.datetime.now() imgname = "missing" greenspace = 0.0 servername = socket.gethostname() if not testing: lsimg = landsatImg.GrassLandsat(gid, cityname, box, coord_sys, location) log("Getting Image List") lsimg.getImgList(pgConn) log("Getting Images from Swift") lsimg.getSwiftImgs() log("Combine into PNG") lsimg.combineIntoPng() log("Read Image Data") lsimg.img.readImgData() # get polygon as list of points (ie, not a string) log("Extracting Polygon points") polygon = wkt_to_list(convex_hull) log("Calculating Greenspace") greenspace = calc_greenspace(lsimg.img, polygon) log("Greenvalue of " ,str(greenspace), " was found") log("RESULT:",gid, cityname, greenspace) # pgConn.createUpdateStmnt(greenspace, # lsimg.gid, lsimg.city, # start_t, end_t, # lsimg.img.imgname, # servname, location) lsimg.uploadToSwift() imgname = lsimg.img.imgname # only in for test tmp_directory = lsimg.file_manager.tmp_file_dir del lsimg if os.path.exists(tmp_directory): print 'Cleanup failed to delete ' + tmp_directory else: print "warning testing mode" end_t = datetime.datetime.now() result_dict = {'id':gid, 'name':cityname, 'greenspace_val':greenspace, 'stime':start_t.isoformat(), 'etime':end_t.isoformat(), 'imgname':imgname, 'servername':servername, 'location':location } return result_dict
def combineIntoPng(self): """ """ def getMostRecentSet(fnames): """ """ dates = [] for f in fnames: # we are always encounter 'dt' before the date fdate = f.split('t')[1].split('.')[0] if fdate not in dates: dates.append(int(fdate)) maxdate = str(max(dates)) newfiles = [] for f in fnames: if maxdate in f: newfiles.append(f) log(" in getMostRecentSet, newfiles are " + " ".join(newfiles)) return newfiles def decompressTiffs(files): """ """ import zlib tiffs = [] cwd = os.getcwd() log("In decompressTiffs, cwd is " + cwd) log("Decompressing files") try: for f in files: # tname = f.rstrip('.gz') tname = self.file_manager.get_tiff_name(f) tiffs.append(tname) # print "In decompress_tiffs: file is " + f + " tname is " + tname if not os.path.exists(tname): # inf = gzip.GzipFile(os.path.join(cwd, f), 'rb') inf = gzip.GzipFile(f, 'rb') dcmpdata = inf.read() inf.close() outf = open(tname, "w") # try catches???? outf.write(dcmpdata) outf.close() except IOError as e: # might be corrupt - delete now so not cached for f in files: print "Corrupt files, unlinking:", f os.unlink(f) raise AssertionError(e) except zlib.error as e: # might be corrupt - delete now so not cached for f in files: print "Corrupt files, unlinking:", f os.unlink(f) raise AssertionError(e) log(" At the end of decompressTiffs, files are: " + " ".join(files) + ". Tiffs are:" + " ".join(tiffs)) log("Decompressing files done") return tiffs assert(self.havefiles) assert(len(self.files)) # get raster 3 band images for each prefix fnames = decompressTiffs(self.files) #print fnames if len(fnames) > 3: log("Combining images") fnames = combine.combine_bands(fnames, self.file_manager, str(self.gid)) log("Getting shapefile for", str(self.gid)) shpname = trim.getShapefile(self.file_manager.tmp_file_dir, self.gid) log("Shapefile is " + shpname) #try: trim.crop(shpname, fnames[0], fnames[1], fnames[2], prefix="trim_", shapefile_tmpDir = self.file_manager.shapefile_tmpDir) dir_name = gcswift.get_dir_name(fnames[0]) fnames = [dir_name + "/trim_"+gcswift.get_raw_file_name(name) for name in fnames] log("New fnames are: " + " ".join(fnames)) #except AssertionError as e: # print e namesAndBands = [(fname, getBand(fname)) for fname in fnames] namesAndBands = sorted(namesAndBands, key=itemgetter(1), reverse=True) fnames = [fname for (fname, band) in namesAndBands] pre = self.file_manager.tmp_file_dir + "/" + str(self.gid) allbandsTIF = pre +"_allbands.tif" allbandsPNG = pre +"_allbands.png" mergeCmd = '/usr/local/bin/gdal_merge.py -n -9999 -a_nodata -9999 -separate -o ' mergeCmd += allbandsTIF for filename in fnames: mergeCmd += ' ' + filename log("creating... " + allbandsTIF + " with "+ mergeCmd) p = Popen(mergeCmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) p.wait() output = p.stdout.read() assert p.returncode == 0, "Creation of merged bands Failed" log(output) # # Now convert to PNG # translateCmd = "/usr/local/bin/gdal_translate -of png -ot Byte -scale %s %s" % (allbandsTIF, allbandsPNG) log("creating... " + allbandsPNG + " with " + translateCmd) p = Popen(translateCmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) p.wait() output = p.stdout.read() assert p.returncode == 0, "Creation of png Failed" log(output) # we give it 1) the name of the png created from grass # and 2) the name the greenspace calc will create # and 3) the directory where they will be stored pngimg = pngImg(allbandsPNG, str(self.gid) + ".png", self.file_manager.tmp_file_dir) pngimg.bbox = self.bbox self.img = pngimg
def getImgList(self,pgConn): """ """ import json sql = "select fname, ST_AsGeoJson(tiff4326.the_geom) "\ "from (select ST_Transform(the_geom, 4326) as the_geom from map where gid = "+str(self.gid)+") as map , tiff4326 where ST_Intersects(map.the_geom, tiff4326.the_geom);" records = pgConn.performSelect(sql) if len(records)==0: raise MissingCoverage("Could not find any images in the database for "+str(self.gid)) decided_date = None self.images = [] images = [] for r in records: is_bad, bb = self.check_bad_record(json.loads(r[1])) if is_bad: log("Warning: Skipping " + r[0] + " because of wraparound bug. Coords: " + str(bb)) continue else: log("Info: Proccessing " + r[0]) pieces = r[0].split('_') if len(pieces) == 4: # we only want 5dt or 7dt for landsat 7 (7),decade set (d) and triband (t) if 'dt' in pieces[1] and checkValidImageSpec(pieces): images.append(imageID(pieces)) else: diagnoseBadSpec(pieces, r[0]) # images1 = [] for image in images: subsumed = reduce(lambda x,y: x or y, [otherImage.subsumes(image) for otherImage in images]) if subsumed: continue self.images.append(image) # images1.append(image) # Make sure each image has a neighbor ## for image in images1: ## hasNeighbor = reduce(lambda x, y: x or y, [otherImage.overlaps(image) for otherImage in images1]) ## if hasNeighbor: self.images.append(image) ## # If there are no connected images, just pick the one with the least path, least ## # row; any choice will do, we just need it to be consistent across bands ## if len(self.images) == 0: ## images1 = sorted(images1, key=attrgetter('path', 'row')) ## self.images = [images1[0]] # print "total images", len(images), "total unsubsumed images", len(images1), "total connected images", len(self.images) log("total images %s total subsumed images %s"%(len(images), len(self.images))) for image in self.images: self.files.append(image.fileName) self.buckets.append((image.bucket, image.fileName)) log( "%s images intersect the city %s" % (len(self.files),self.city)) log( "At the end of getImgs, files are " + " ".join(self.files)) return len(self.files)