Beispiel #1
0
def main():
  db_cas = ColumnFamilyCass(database_cas.keyspace, database_cas.cf_name, database_cas.nodelist) 

  # Does the user want to empty the column family?
  if 'empty' in sys.argv:
    print "Purging data." 
    db_cas.purge()
    print "Done..."
    exit()

  try:
      continent = '_'.join(map(lambda s: s.capitalize(), re.split('[ _]', sys.argv[1])))
  except: 
      print "Please specify the continent. Africa, Australia, Eurasia, Islands, North_America or South_America."
      sys.exit(1)

  [north, south, west, east] = util.getBoundingBox(sys.argv, 3)

  files_hashes = util.getFilesHashes(continent)

  number_of_tiles = util.numberOfFiles(files_hashes, north, south, west, east)

  # Verify result?
  if 'verify' in sys.argv:
    verify(db_cas, number_of_tiles, files_hashes, continent,  north, south, west, east)
    #@todo how does this works?

  # If a tile name is given as the second argument it will resume from there.
  p = re.compile('[NSEW]\d*')
  resume_from = ""
  try:
    if(p.find(sys.argv[2])):
      resume_from = sys.argv[2]

  except: 
    None

  i = 0

  for file in files_hashes:
    # Strip .hgt.zip extension:
    file = file[1][0:-8] 
    # Get latitude and longitude from file name 
    [lat,lon] = util.getLatLonFromFileName(file)

    if util.inBoundingBox(lat, lon, north, south, west, east):
      i = i + 1

      # Are we resuming?
      if resume_from == file:
        resume_from = "" 

      if resume_from == "":

        # First check if the tile is not already in the database:
        try:
          db_cas.fetchTopLeftAltitude(lat, lon)

          print("Skipping tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...")

        except IndexError:
          print("Insert data for tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...")

          # Load tile from file
          tile = loadTile(continent, file)

          db_cas.insertTile(tile, lat, lon)

  print("All tiles inserted. You will want to run a nodetool repair.")

  print("Import is done.  Pleasy verify the result with python read_data_cas.py verify")
Beispiel #2
0
# python test/verify_download.py Australia
# python test/verify_download.py Eurasia 54 47 0 16

# I could not find the 'official' MD5 checksums for the data so I created them 
# myself. So if you get an error message, that could also mean the MD5 
# checksum in this script is wrong. In that case: please let me know.

import hashlib
import sys
import os
import re

sys.path += [os.path.abspath('.')]
from data import util, files

continent = '_'.join(map(lambda s: s.capitalize(), re.split('[ _]', sys.argv[1])))

[north, south, west, east] = util.getBoundingBox(sys.argv, 2)

files_hashes = util.getFilesHashes(continent)

if __name__ == '__main__':
  for file_hash in files_hashes:
    [lat,lon] = util.getLatLonFromFileName(file_hash[1])
    if util.inBoundingBox(lat, lon, north, south, west, east):
      f = file("data/" + continent + "/" + file_hash[1])
      if(not hashlib.md5(f.read()).hexdigest() == file_hash[0]):
        print "Error in file " + file_hash[1]
        exit()
  
def main():
  db_pg = DatabasePg(database_pg.db, database_pg.db_user, database_pg.db_pass)
  db_psycopg2 = DatabasePsycopg2(database_pg.db, database_pg.db_user, database_pg.db_pass)

  try:
      continent = sys.argv[1]
      
  except: 
      print "Please specify the continent. Africa, Australia, Eurasia, Islands, North_America or South_America."

  # Does the user want to empty the database?
  if 'empty' in sys.argv:
    print "Deleting tables from databse..." 
    db_pg.dropAllTables()
    print "Done..."
    exit()

  [north, south, west, east] = util.getBoundingBox(sys.argv, 3)
      
  files_hashes = util.getFilesHashes(continent)
  
  number_of_tiles = util.numberOfFiles(files_hashes, north, south, west, east)
  
  # Verify result?
  if 'verify' in sys.argv:
    verify(db_pg, number_of_tiles, files_hashes, continent,  north, south, west, east)

  # If a tile name is given as the second argument it will resume from there.
  p = re.compile('[NSEW]\d*')
  resume_from = ""
  try:
    if(p.find(sys.argv[2])):
      resume_from = sys.argv[2]
      
  except: 
    None
    
  db_pg.createTableAltitude()

  i = 0

  for file in files_hashes:
    # Strip .hgt.zip extension:
    file = file[1][0:-8] 
    # Get latitude and longitude from file name 
    [lat,lon] = util.getLatLonFromFileName(file)
    
    if util.inBoundingBox(lat, lon, north, south, west, east):
      i = i + 1
  
      # Are we resuming?
      if resume_from == file:
        resume_from = "" 
      
      if resume_from == "":

        # Load tile from file
        tile = loadTile(continent, file)

        # First check if the tile is not already in the database:

        try:
          db_pg.fetchTopLeftAltitude(lat, lon)
          print("Skipping tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...")

        except:
          print("Insert data for tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...")

          db_psycopg2.insertTile(tile, lat, lon)

  print("All tiles inserted. Pleasy verify the result with python \
  read_data.py verify")