def main(): db_cas = ColumnFamilyCass(database_cas.keyspace, database_cas.cf_name, database_cas.nodelist) # Does the user want to empty the column family? if 'empty' in sys.argv: print "Purging data." db_cas.purge() print "Done..." exit() try: continent = '_'.join(map(lambda s: s.capitalize(), re.split('[ _]', sys.argv[1]))) except: print "Please specify the continent. Africa, Australia, Eurasia, Islands, North_America or South_America." sys.exit(1) [north, south, west, east] = util.getBoundingBox(sys.argv, 3) files_hashes = util.getFilesHashes(continent) number_of_tiles = util.numberOfFiles(files_hashes, north, south, west, east) # Verify result? if 'verify' in sys.argv: verify(db_cas, number_of_tiles, files_hashes, continent, north, south, west, east) #@todo how does this works? # If a tile name is given as the second argument it will resume from there. p = re.compile('[NSEW]\d*') resume_from = "" try: if(p.find(sys.argv[2])): resume_from = sys.argv[2] except: None i = 0 for file in files_hashes: # Strip .hgt.zip extension: file = file[1][0:-8] # Get latitude and longitude from file name [lat,lon] = util.getLatLonFromFileName(file) if util.inBoundingBox(lat, lon, north, south, west, east): i = i + 1 # Are we resuming? if resume_from == file: resume_from = "" if resume_from == "": # First check if the tile is not already in the database: try: db_cas.fetchTopLeftAltitude(lat, lon) print("Skipping tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...") except IndexError: print("Insert data for tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...") # Load tile from file tile = loadTile(continent, file) db_cas.insertTile(tile, lat, lon) print("All tiles inserted. You will want to run a nodetool repair.") print("Import is done. Pleasy verify the result with python read_data_cas.py verify")
# python test/verify_download.py Australia # python test/verify_download.py Eurasia 54 47 0 16 # I could not find the 'official' MD5 checksums for the data so I created them # myself. So if you get an error message, that could also mean the MD5 # checksum in this script is wrong. In that case: please let me know. import hashlib import sys import os import re sys.path += [os.path.abspath('.')] from data import util, files continent = '_'.join(map(lambda s: s.capitalize(), re.split('[ _]', sys.argv[1]))) [north, south, west, east] = util.getBoundingBox(sys.argv, 2) files_hashes = util.getFilesHashes(continent) if __name__ == '__main__': for file_hash in files_hashes: [lat,lon] = util.getLatLonFromFileName(file_hash[1]) if util.inBoundingBox(lat, lon, north, south, west, east): f = file("data/" + continent + "/" + file_hash[1]) if(not hashlib.md5(f.read()).hexdigest() == file_hash[0]): print "Error in file " + file_hash[1] exit()
def main(): db_pg = DatabasePg(database_pg.db, database_pg.db_user, database_pg.db_pass) db_psycopg2 = DatabasePsycopg2(database_pg.db, database_pg.db_user, database_pg.db_pass) try: continent = sys.argv[1] except: print "Please specify the continent. Africa, Australia, Eurasia, Islands, North_America or South_America." # Does the user want to empty the database? if 'empty' in sys.argv: print "Deleting tables from databse..." db_pg.dropAllTables() print "Done..." exit() [north, south, west, east] = util.getBoundingBox(sys.argv, 3) files_hashes = util.getFilesHashes(continent) number_of_tiles = util.numberOfFiles(files_hashes, north, south, west, east) # Verify result? if 'verify' in sys.argv: verify(db_pg, number_of_tiles, files_hashes, continent, north, south, west, east) # If a tile name is given as the second argument it will resume from there. p = re.compile('[NSEW]\d*') resume_from = "" try: if(p.find(sys.argv[2])): resume_from = sys.argv[2] except: None db_pg.createTableAltitude() i = 0 for file in files_hashes: # Strip .hgt.zip extension: file = file[1][0:-8] # Get latitude and longitude from file name [lat,lon] = util.getLatLonFromFileName(file) if util.inBoundingBox(lat, lon, north, south, west, east): i = i + 1 # Are we resuming? if resume_from == file: resume_from = "" if resume_from == "": # Load tile from file tile = loadTile(continent, file) # First check if the tile is not already in the database: try: db_pg.fetchTopLeftAltitude(lat, lon) print("Skipping tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...") except: print("Insert data for tile " + file + " (" + str(i) + " / " + str(number_of_tiles) + ") ...") db_psycopg2.insertTile(tile, lat, lon) print("All tiles inserted. Pleasy verify the result with python \ read_data.py verify")