Example #1
0
def verify(db, number_of_tiles, files_hashes, continent, north, south, west, east):
    # For every tile, verify the bottom left coordinate.
    for file in files_hashes:
      # Strip .hgt.zip extension:
      file = file[1][0:-8] 
    
      [lat,lon] = util.getLatLonFromFileName(file)
      
      # Only a smaller part of Australia (see below):
      if util.inBoundingBox(lat, lon, north, south, west, east):
      
        print "Verify " + file + "..." 
    
        # Get top left altitude from file:
        coordinate_file = loadTile(continent, file)[1][0]
    
        # Get top left altitude from database:
        coordinate_db = db.fetchTopLeftAltitude(lat,lon)
        
        if coordinate_db != coordinate_file:
          print "Mismatch tile " + file[1]
          exit() 
    
    # Check the total number of points in the database:
    
    print "Check the total number of points in the database..."
    
    total = db.query("SELECT count(pos) FROM altitude")[0][0]
    if not total == number_of_tiles * 1200 * 1200:
      print "Not all tiles have been (completely) inserted!"
      exit()
        
    print "All tiles seem to have made it into the database! Enjoy."
    
    exit()
  def testInsertTileIntoDatabase(self):

    # Load example tile
    fulltile = loadTile('Australia', 'S37E145')
    tile = []
    for row in fulltile[0:11]:
      tile.append(row[0:11])

    # Get lat and lon from filename
    [lat,lon] = getLatLonFromFileName("S37E145")

    # Make the tile smaller, so this will be faster:
    # 11x11 tile: because the top row and right column are dropped,
    # only the bottom-left 10x10 tile will be stored in the
    # database.

    # Insert tile into our Cassandra ColumnFamily
    self.db_cas.insertTile(tile, lat, lon)

    # Check if the tile is indeed in the ColumnFamily
    tile_back = self.db_cas.readTile(lat, lon)

    for i in range(len(tile) - 1):
      for j in range(len(tile) - 1):
        self.assert_(tile_back[i][j] == tile[i+1][j])
  def testInsertTileIntoDatabase(self):
    # Create table
    self.assert_(self.db_psycopg2.createTableAltitude())

    # Load example tile
    fulltile = loadTile('Australia', 'S37E145')
    tile = []
    for row in fulltile[0:11]:
      tile.append(row[0:11])

    # Get lat and lon from filename
    [lat,lon] = getLatLonFromFileName("S37E145")

    # Make the tile smaller, so this will be faster:
    # 11x11 tile: because the top row and right column are dropped,
    # only the bottom-left 10x10 tile will be stored in the
    # database.

    # Insert tile into database
    # We use psycopg2 for the connection in this case.
    self.db_psycopg2.insertTile(tile, lat, lon)

    # Check if the tile is indeed in the database
    tile_back = self.db_psycopg2.readTile(lat, lon)
    for i in range(len(tile) - 1):
      for j in range(len(tile) - 1):
        self.assert_(tile_back[i][j] == tile[i+1][j])
  def testInsertTileWithNull(self):
    # Create table
    self.assert_(self.db_psycopg2.createTableAltitude())

    # Some tiles contain the value -32768, which means NULL (not implemented yet)
    # Tile S27E123 has several -32768 values, for example tile[1086][462]
    fulltile = loadTile('Australia', 'S27E123')
    self.assertEqual(fulltile[1086][462], -32768)

    # Take part of the tile around that area
    tile = []
    for row in fulltile[1080:1091]:
      tile.append(row[460:471])

    # Get lat and lon from filename
    [lat,lon] = getLatLonFromFileName("S27E123")

    # Insert tile into database
    self.db_psycopg2.insertTile(tile, lat, lon)

    # Check if the tile is indeed in the database
    tile_back = self.db_psycopg2.readTile(lat, lon)
    for i in range(len(tile) - 1):
      for j in range(len(tile) - 1):
        self.assert_(tile_back[i][j] == tile[i+1][j])
Example #5
0
def main():
    # First we make a list of all files that need to be download. This depends
    # on the arguments given to the program.
    # The first argument should be the continent:
    # * Africa
    # * Australia
    # * Eurasia
    # * Islands
    # * North_America
    # * South_America

    try:
        continent = "_".join(map(lambda s: s.capitalize(), re.split("[ _]", sys.argv[1])))
    except:
        continent = ""
    if not continent in "Africa Australia Eurasia Islands North_America South_America".split():
        usage()
        sys.exit(1)

    # First we get the list of files through an HTTP connection.
    http = httplib.HTTPConnection("dds.cr.usgs.gov")

    http.request("GET", "/srtm/version2_1/SRTM3/" + continent + "/")

    # Now list all tiles of that continent.
    # See http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/[continent]/

    response = http.getresponse()
    html = response.read()

    files = re.findall(r'<a href="([^"]+)"', html)
    files.pop(0)  # remove "Parent Directory" link

    # And close connection.

    http.close()

    # Now download all files using urllib.urlretrieve

    # Do we have a bounding box?
    [north, south, west, east] = util.getBoundingBox(sys.argv, 2)
    for i in range(len(files)):
        if not (os.path.exists("data/" + continent + "/" + files[i])):
            [lat, lon] = util.getLatLonFromFileName(files[i])
            if util.inBoundingBox(lat, lon, north, south, west, east):
                print "Downloading " + files[i] + " (lat = " + str(lat) + " , lon = " + str(lon) + " )... (" + str(
                    i + 1
                ) + " of " + str(len(files)) + ")"
                (f, tmp) = tempfile.mkstemp()
                try:
                    urllib.urlretrieve(
                        "http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/" + continent + "/" + files[i], tmp
                    )
                    os.close(f)
                    shutil.move(tmp, "data/" + continent + "/" + files[i])
                except Exception, msg:
                    sys.stderr.write(str(msg) + "\n")
                    os.remove(tmp)
Example #6
0
def main():
    # First we make a list of all files that need to be download. This depends
    # on the arguments given to the program.
    # The first argument should be the continent:
    # * Africa  
    # * Australia  
    # * Eurasia  
    # * Islands  
    # * North_America  
    # * South_America    

    try:
        continent = '_'.join(map(lambda s: s.capitalize(), re.split('[ _]', sys.argv[1])))
    except:
        continent = ""
    if not continent in "Africa Australia Eurasia Islands North_America South_America".split():
        usage()
        sys.exit(1)
        
    # First we get the list of files through an HTTP connection.
    http = httplib.HTTPConnection('dds.cr.usgs.gov')

    http.request("GET", "/srtm/version2_1/SRTM3/" + continent + "/");

    # Now list all tiles of that continent.
    # See http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/[continent]/
    
    response = http.getresponse()
    html = response.read()

    files = re.findall(r'<a href="([^"]+)"', html)
    files.pop(0) #remove "Parent Directory" link
    
    # And close connection.
    
    http.close()

    # Now download all files using urllib.urlretrieve
    
    # Do we have a bounding box?
    [north, south, west, east] = util.getBoundingBox(sys.argv, 2)
    for i in range(len(files)):
      if not (os.path.exists("data/" + continent + "/" + files[i])):
          [lat,lon] = util.getLatLonFromFileName(files[i])
          if util.inBoundingBox(lat, lon, north, south, west, east):
            print "Downloading " + files[i] + " (lat = " + str(lat)  + " , lon = " + str(lon) + " )... (" + str(i + 1) + " of " + str(len(files)) +")"
            (f, tmp) = tempfile.mkstemp()
            try:
                urllib.urlretrieve("http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/" + continent + "/"  + files[i], tmp)
                os.close(f)
                shutil.move(tmp, "data/" + continent + "/" + files[i])
            except Exception, msg:
                sys.stderr.write(str(msg) + "\n")
                os.remove(tmp)
Example #7
0
def verify(db, number_of_tiles, files_hashes, continent, north, south, west,
           east):
    # For every tile, verify the bottom left coordinate.
    for file in files_hashes:
        # Strip .hgt.zip extension:
        file = file[1][0:-8]

        [lat, lon] = util.getLatLonFromFileName(file)

        # Only a smaller part of Australia (see below):
        if util.inBoundingBox(lat, lon, north, south, west, east):

            print "Verify " + file + "..."

            # Get top left altitude from file:
            coordinate_file = loadTile(continent, file)[1][0]

            # Get top left altitude from database:
            coordinate_db = db.fetchTopLeftAltitude(lat, lon)

            if coordinate_db != coordinate_file:
                print "Mismatch tile " + file[1]
                exit()

    # Check the total number of points in the database:

    print "Check the total number of points in the database..."

    sql = db.query("SELECT count(pos) FROM altitude")
    total = int(sql.getresult()[0][0])
    if not total == number_of_tiles * 1200 * 1200:
        print "Not all tiles have been (completely) inserted!"
        exit()

    print "All tiles seem to have made it into the database! Enjoy."

    exit()
            f.write(str(\
            begin + (row-1) * 1200 + col\
            ) + ", " + str(tile[row][col] ) + "\n")

    f.close()


if __name__ == '__main__':
    # We will only upload 1 tile to the Google App Engine. This will take quite
    # a bit of time. For the offline data store, we will only "upload" the city
    # of Heidelberg; the offline data store is very slow.

    # For this we need tile N49E008.
    name = "N49E008"
    tile = loadTile("Eurasia", name)
    [lat, lon] = util.getLatLonFromFileName(name)

    if not ("online" in sys.argv or "offline" in sys.argv):
        print "Online or offline?"
        exit()

    if sys.argv[1] == "offline":
        # If we are offline, we'll only look the center of Heidelberg.
        # 49.39 --- 49.42
        # 8.67 --- 8.71
        # That corresponds to:
        row_top = int((1.0 - 0.42) * 1200.)
        row_bottom = int((1.0 - 0.39) * 1200.)
        col_left = int(0.67 * 1200.)
        col_right = int(0.71 * 1200.)
        # So that 1813 records
Example #9
0
# python test/verify_download.py Australia
# python test/verify_download.py Eurasia 54 47 0 16

# I could not find the 'official' MD5 checksums for the data so I created them 
# myself. So if you get an error message, that could also mean the MD5 
# checksum in this script is wrong. In that case: please let me know.

import hashlib
import sys
import os
import re

sys.path += [os.path.abspath('.')]
from data import util, files

continent = '_'.join(map(lambda s: s.capitalize(), re.split('[ _]', sys.argv[1])))

[north, south, west, east] = util.getBoundingBox(sys.argv, 2)

files_hashes = util.getFilesHashes(continent)

if __name__ == '__main__':
  for file_hash in files_hashes:
    [lat,lon] = util.getLatLonFromFileName(file_hash[1])
    if util.inBoundingBox(lat, lon, north, south, west, east):
      f = file("data/" + continent + "/" + file_hash[1])
      if(not hashlib.md5(f.read()).hexdigest() == file_hash[0]):
        print "Error in file " + file_hash[1]
        exit()
  
Example #10
0
def main():
    # First we make a list of all files that need to be download. This depends
    # on the arguments given to the program.
    # The first argument should be the continent:
    # * Africa  
    # * Australia  
    # * Eurasia  
    # * Islands  
    # * North_America  
    # * South_America    

    if len(sys.argv) > 1:
        continent = sys.argv[1]
        util.verifyIsContinent(continent)
    else:
        print "Please provide arguments \n",\
        "First argument should be Africa, Australia, Eurasia, Islands, North_America or South_America.\n",\
        "Second argument (optional) specifies from which tile to resume. Use full file name e.g. \n",\
        "'N36W004.hgt.zip'. Set to 0 start at the first file. \n",\
        "Argument 3-6 optionally specify a bounding box: north, south, west, east"
        exit()
        
    # First we get the list of files through an FTP connection.
    ftp = FTP('e0srp01u.ecs.nasa.gov')

    ftp.login()

    ftp.cwd("srtm/version2/SRTM3/" + continent)

    # Now list all tiles of that continent.
    # See ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/[continent]/
    
    files = ftp.nlst()
    
    # And close connection.
    
    ftp.close()

    # Now download all files using urllib.urlretrieve
    
    # Determine if we need to resume at a certain point
    if(len(sys.argv) > 2):
        resume = sys.argv[2]
        if not(resume == "0"):
            skip = True     
            print "Resume from " + resume + "..."
        else:
            skip = False
    else: 
        skip = False

    # Do we have a bounding box?
    [north, south, west, east] = util.getBoundingBox(sys.argv, 3)
    for i in range(len(files)):
      if skip:
          if files[i] == resume:
              skip = False
          
      if not(skip):
          [lat,lon] = util.getLatLonFromFileName(files[i])
          if util.inBoundingBox(lat, lon, north, south, west, east):
            print "Downloading " + files[i] + " (lat = " + str(lat)  + " , lon = " + str(lon) + " )... (" + str(i + 1) + " of " + str(len(files)) +")"
            urllib.urlretrieve("ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/" + continent + "/"  + files[i],"data/" + continent + "/" + files[i])
Example #11
0
def main():
    # First we make a list of all files that need to be download. This depends
    # on the arguments given to the program.
    # The first argument should be the continent:
    # * Africa  
    # * Australia  
    # * Eurasia  
    # * Islands  
    # * North_America  
    # * South_America    

    if len(sys.argv) > 1:
        continent = sys.argv[1]
        util.verifyIsContinent(continent)
    else:
        print "Please provide arguments \n",\
        "First argument should be Africa, Australia, Eurasia, Islands, North_America or South_America.\n",\
        "Second argument (optional) specifies from which tile to resume. Use full file name e.g. \n",\
        "'N36W004.hgt.zip'. Set to 0 start at the first file. \n",\
        "Argument 3-6 optionally specify a bounding box: north, south, west, east"
        exit()
        
    # First we get the list of files through an HTTP connection.
    http = httplib.HTTPConnection('dds.cr.usgs.gov')

    http.request("GET", "/srtm/version2_1/SRTM3/" + continent + "/");

    # Now list all tiles of that continent.
    # See http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/[continent]/
    
    response = http.getresponse()
    html = response.read()

    files = re.findall(r'<a href="([^"]+)"', html)
    files.pop(0) #remove "Parent Directory" link
    
    # And close connection.
    
    http.close()

    # Now download all files using urllib.urlretrieve
    
    # Determine if we need to resume at a certain point
    if(len(sys.argv) > 2):
        resume = sys.argv[2]
        if not(resume == "0"):
            skip = True     
            print "Resume from " + resume + "..."
        else:
            skip = False
    else: 
        skip = False

    # Do we have a bounding box?
    [north, south, west, east] = util.getBoundingBox(sys.argv, 3)
    for i in range(len(files)):
      if skip:
          if files[i] == resume:
              skip = False
          
      if not(skip):
          [lat,lon] = util.getLatLonFromFileName(files[i])
          if util.inBoundingBox(lat, lon, north, south, west, east):
            print "Downloading " + files[i] + " (lat = " + str(lat)  + " , lon = " + str(lon) + " )... (" + str(i + 1) + " of " + str(len(files)) +")"
            urllib.urlretrieve("http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/" + continent + "/"  + files[i],"data/" + continent + "/" + files[i])
Example #12
0
 def testGetLatLonFromFileName(self):
   self.assertEqual([-11,119], getLatLonFromFileName("S11E119"))
   self.assertEqual([11,119], getLatLonFromFileName("N11E119"))
   self.assertEqual([11,-119], getLatLonFromFileName("N11W119"))
   self.assertEqual([-11,-119], getLatLonFromFileName("S11W119"))
    for col in range(left_col, right_col + 1):
      f.write(str(\
      begin + (row-1) * 1200 + col\
      ) + ", " + str(tile[row][col] ) + "\n")

  f.close() 

if __name__ == '__main__':
  # We will only upload 1 tile to the Google App Engine. This will take quite 
  # a bit of time. For the offline data store, we will only "upload" the city 
  # of Heidelberg; the offline data store is very slow.
  
  # For this we need tile N49E008. 
  name = "N49E008"
  tile = loadTile("Eurasia", name)
  [lat,lon] = util.getLatLonFromFileName(name)

  if not ("online" in sys.argv or "offline" in sys.argv):
      print "Online or offline?"
      exit()
  
  if sys.argv[1] == "offline":
      # If we are offline, we'll only look the center of Heidelberg.
      # 49.39 --- 49.42
      # 8.67 --- 8.71
      # That corresponds to:
      row_top = int((1.0 - 0.42) * 1200.)
      row_bottom = int((1.0 - 0.39) * 1200.)
      col_left = int(0.67 * 1200.)
      col_right = int(0.71 * 1200.)
      # So that 1813 records
Example #14
0
def main():
    # First we make a list of all files that need to be download. This depends
    # on the arguments given to the program.
    # The first argument should be the continent:
    # * Africa
    # * Australia
    # * Eurasia
    # * Islands
    # * North_America
    # * South_America

    if len(sys.argv) > 1:
        continent = sys.argv[1]
        util.verifyIsContinent(continent)
    else:
        print "Please provide arguments \n",\
        "First argument should be Africa, Australia, Eurasia, Islands, North_America or South_America.\n",\
        "Second argument (optional) specifies from which tile to resume. Use full file name e.g. \n",\
        "'N36W004.hgt.zip'. Set to 0 start at the first file. \n",\
        "Argument 3-6 optionally specify a bounding box: north, south, west, east"
        exit()

    # First we get the list of files through an FTP connection.
    ftp = FTP('e0srp01u.ecs.nasa.gov')

    ftp.login()

    ftp.cwd("srtm/version2/SRTM3/" + continent)

    # Now list all tiles of that continent.
    # See ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/[continent]/

    files = ftp.nlst()

    # And close connection.

    ftp.close()

    # Now download all files using urllib.urlretrieve

    # Determine if we need to resume at a certain point
    if (len(sys.argv) > 2):
        resume = sys.argv[2]
        if not (resume == "0"):
            skip = True
            print "Resume from " + resume + "..."
        else:
            skip = False
    else:
        skip = False

    # Do we have a bounding box?
    [north, south, west, east] = util.getBoundingBox(sys.argv, 3)
    for i in range(len(files)):
        if skip:
            if files[i] == resume:
                skip = False

        if not (skip):
            [lat, lon] = util.getLatLonFromFileName(files[i])
            if util.inBoundingBox(lat, lon, north, south, west, east):
                print "Downloading " + files[i] + " (lat = " + str(
                    lat) + " , lon = " + str(lon) + " )... (" + str(
                        i + 1) + " of " + str(len(files)) + ")"
                urllib.urlretrieve(
                    "ftp://e0srp01u.ecs.nasa.gov/srtm/version2/SRTM3/" +
                    continent + "/" + files[i],
                    "data/" + continent + "/" + files[i])
 def testGetLatLonFromFileName(self):
   self.assertEqual([-11,119], getLatLonFromFileName("S11E119"))
   self.assertEqual([11,119], getLatLonFromFileName("N11E119"))
   self.assertEqual([11,-119], getLatLonFromFileName("N11W119"))
   self.assertEqual([-11,-119], getLatLonFromFileName("S11W119"))
Example #16
0
def main():
  db_pg = DatabasePg(database_pg.db, database_pg.db_user, database_pg.db_pass)
  db_psycopg2 = DatabasePsycopg2(database_pg.db, database_pg.db_user, database_pg.db_pass)

  try:
      continent = sys.argv[1]
      
  except: 
      print "Please specify the continent. Africa, Australia, Eurasia, Islands, North_America or South_America."

  # Does the user want to empty the database?
  if 'empty' in sys.argv:
    print "Deleting tables from databse..." 
    db_pg.dropAllTables()
    print "Done..."
    exit()

  [north, south, west, east] = util.getBoundingBox(sys.argv, 3)
      
  files_hashes = util.getFilesHashes(continent)
  
  number_of_tiles = util.numberOfFiles(files_hashes, north, south, west, east)
  
  # Verify result?
  if 'verify' in sys.argv:
    verify(db_pg, number_of_tiles, files_hashes, continent,  north, south, west, east)

  # If a tile name is given as the second argument it will resume from there.
  p = re.compile('[NSEW]\d*')
  resume_from = ""
  try:
    if(p.find(sys.argv[2])):
      resume_from = sys.argv[2]
      
  except: 
    None
    
  db_pg.createTableAltitude()

  i = 0

  for file in files_hashes:
    # Strip .hgt.zip extension:
    file = file[1][0:-8] 
    # Get latitude and longitude from file name 
    [lat,lon] = util.getLatLonFromFileName(file)
    
    if util.inBoundingBox(lat, lon, north, south, west, east):
      i = i + 1
  
      # Are we resuming?
      if resume_from == file:
        resume_from = "" 
      
      if resume_from == "":

        # Load tile from file
        tile = loadTile(continent, file)

        # First check if the tile is not already in the database:

        try:
          db_pg.fetchTopLeftAltitude(lat, lon)
          print("Skipping tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...")

        except:
          print("Insert data for tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...")

          db_psycopg2.insertTile(tile, lat, lon)

  print("All tiles inserted. Pleasy verify the result with python \
  read_data.py verify")
Example #17
0
def main():
  db_cas = ColumnFamilyCass(database_cas.keyspace, database_cas.cf_name, database_cas.nodelist) 

  # Does the user want to empty the column family?
  if 'empty' in sys.argv:
    print "Purging data." 
    db_cas.purge()
    print "Done..."
    exit()

  try:
      continent = '_'.join(map(lambda s: s.capitalize(), re.split('[ _]', sys.argv[1])))
  except: 
      print "Please specify the continent. Africa, Australia, Eurasia, Islands, North_America or South_America."
      sys.exit(1)

  [north, south, west, east] = util.getBoundingBox(sys.argv, 3)

  files_hashes = util.getFilesHashes(continent)

  number_of_tiles = util.numberOfFiles(files_hashes, north, south, west, east)

  # Verify result?
  if 'verify' in sys.argv:
    verify(db_cas, number_of_tiles, files_hashes, continent,  north, south, west, east)
    #@todo how does this works?

  # If a tile name is given as the second argument it will resume from there.
  p = re.compile('[NSEW]\d*')
  resume_from = ""
  try:
    if(p.find(sys.argv[2])):
      resume_from = sys.argv[2]

  except: 
    None

  i = 0

  for file in files_hashes:
    # Strip .hgt.zip extension:
    file = file[1][0:-8] 
    # Get latitude and longitude from file name 
    [lat,lon] = util.getLatLonFromFileName(file)

    if util.inBoundingBox(lat, lon, north, south, west, east):
      i = i + 1

      # Are we resuming?
      if resume_from == file:
        resume_from = "" 

      if resume_from == "":

        # First check if the tile is not already in the database:
        try:
          db_cas.fetchTopLeftAltitude(lat, lon)

          print("Skipping tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...")

        except IndexError:
          print("Insert data for tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...")

          # Load tile from file
          tile = loadTile(continent, file)

          db_cas.insertTile(tile, lat, lon)

  print("All tiles inserted. You will want to run a nodetool repair.")

  print("Import is done.  Pleasy verify the result with python read_data_cas.py verify")