def _getStringList(self, thisTile):
conn = self._getS3Connection()
bucketRef = None
keyID = None
try:
bucketRef = conn.get_bucket(self.bucketPrefix)
except (AWSConnectionError, Exception) as e:
import traceback, utils
utils.logError(traceback.format_exc())
errStr = "Opening key %s" % (keyID)
raise AppError(utils.timestampStr(), "DataStore", errStr, e)
strEdges = None
tileID = thisTile.getID()
keyRef = None
try:
keyRef = bucketRef.get_key(thisTile.getID())
strEdges = keyRef.get_contents_as_string()
except (AWSConnectionError, Exception) as e:
import traceback, utils
utils.logError(traceback.format_exc())
errStr = "Reading bucket key: %s" % (tileID)
raise AppError(utils.timestampStr(), "DataStore", errStr, e)
resultList = []
someLines = strEdges.split("\n")
listLen = len(someLines)
count = 0
while len(someLines) > 0:
aLine = someLines.pop()
if len(aLine) > 0:
resultList.append(aLine)
conn.close()
return resultList
def _getStringList (self, thisTile):
conn = self._getS3Connection ()
bucketRef = None
keyID = None
try:
bucketRef =conn.get_bucket ( self.bucketPrefix );
except (AWSConnectionError, Exception) as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
errStr = 'Opening key %s' %( keyID )
raise AppError (utils.timestampStr (), 'DataStore', errStr, e )
strEdges = None
tileID = thisTile.getID ()
keyRef = None
try:
keyRef = bucketRef.get_key ( thisTile.getID ())
strEdges = keyRef.get_contents_as_string()
except (AWSConnectionError, Exception) as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
errStr = 'Reading bucket key: %s' %( tileID )
raise AppError (utils.timestampStr (), 'DataStore', errStr, e )
resultList = []
someLines = strEdges.split ('\n')
listLen = len ( someLines )
count = 0
while (len ( someLines ) > 0 ):
aLine = someLines.pop ()
if len (aLine ) > 0:
resultList.append ( aLine )
conn.close ()
return resultList
def _getS3Connection (self):
if 'AWS_ACCESS_KEY_ID' in os.environ:
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
else :
errStr ="AWS_ACCESS_KEY_ID not set"
raise AppError (utils.timestampStr (), 'DataStore', errStr, '' )
if 'AWS_SECRET_ACCESS_KEY' in os.environ:
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
else :
errStr ="AWS_SECRET_ACCESS_KEY not set"
raise AppError (utils.timestampStr (), 'DataStore', errStr, '' )
try:
conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
except (AWSConnectionError, Exception) as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'DataStore', 'creating AWS connection', e )
else:
return conn
def _getS3Connection(self):
if "AWS_ACCESS_KEY_ID" in os.environ:
AWS_ACCESS_KEY_ID = os.environ["AWS_ACCESS_KEY_ID"]
else:
errStr = "AWS_ACCESS_KEY_ID not set"
raise AppError(utils.timestampStr(), "DataStore", errStr, "")
if "AWS_SECRET_ACCESS_KEY" in os.environ:
AWS_SECRET_ACCESS_KEY = os.environ["AWS_SECRET_ACCESS_KEY"]
else:
errStr = "AWS_SECRET_ACCESS_KEY not set"
raise AppError(utils.timestampStr(), "DataStore", errStr, "")
try:
conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
except (AWSConnectionError, Exception) as e:
import traceback, utils
utils.logError(traceback.format_exc())
raise AppError(utils.timestampStr(), "DataStore", "creating AWS connection", e)
else:
return conn
def getKeys (self):
'''
Return a list of all of the Tile (key)s currently held in the repository
'''
lst= []
try:
for key,val in self.iteritems ():
lst.append ( key )
except Exception as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'GraphRepository', \
'Iterating repository, at key %s:' %(key), e )
return lst
def loadEdgeGraphForTile (self, thisTile):
'''
A graph G is defined so:
G = { nodeA: { nodeB : cost_A-B, nodeC: cost_A-C },
nodeB: { nodeA : cost_B-A },
nodeC: { nodeC : cost_C-A }
}
where cost_X_Y is in an instance of DataStructures.Edge
The graph created here will be a subclass of:
- dict
- gis.Tile
'''
try:
strList = self._getStringList ( thisTile )
graph = {}
for thisLine in strList:
thisEdge = self._createEdgeFromLine (thisLine)
# note, do not implement reverse costs/Edges at this
# stage, to reduce memory requirement
graph.setdefault ( thisEdge.sourceNode, {} )
graph[thisEdge.sourceNode][thisEdge.targetNode] = thisEdge
graph.setdefault ( thisEdge.targetNode, {} )
graph[thisEdge.targetNode][thisEdge.sourceNode] = thisEdge
except (AWSConnectionError, Exception) as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'DataStore', \
'loadEdgeGraphForTile', e )
return graph
def _getStringList(self, thisTile):
"""
Search for lines in the file that are in Tile thisTile
return a list of those matched lines.
"""
lstLines = []
try:
fs = open(self.filePath, "r")
if self.hasHeader:
fs.readline()
allLines = fs.readlines()
for thisLine in allLines:
# print thisLine
cols = thisLine.split("|")
centroidWKT = cols[8]
# centroidWKT is like "POINT (0.2343 0.2332432)"
tmpList = centroidWKT.replace("POINT(", "").replace(")", "").split(" ")
X = float(tmpList[0])
Y = float(tmpList[1])
# Find the integer "floor" value of x & y co-ordinates
xVal = int(math.floor(X))
yVal = int(math.floor(Y))
if Tile(xVal, yVal) == thisTile:
lstLines.append(thisLine)
fs.close()
except IOError as e:
import traceback, utils
utils.logError(traceback.format_exc())
raise AppError(utils.timestampStr(), "FileDataStore", "open/read file", e)
return lstLines
def loadEdgeGraphForTile(self, thisTile):
"""
A graph G is defined so:
G = { nodeA: { nodeB : cost_A-B, nodeC: cost_A-C },
nodeB: { nodeA : cost_B-A },
nodeC: { nodeC : cost_C-A }
}
where cost_X_Y is in an instance of DataStructures.Edge
The graph created here will be a subclass of:
- dict
- gis.Tile
"""
try:
strList = self._getStringList(thisTile)
graph = {}
for thisLine in strList:
thisEdge = self._createEdgeFromLine(thisLine)
# note, do not implement reverse costs/Edges at this
# stage, to reduce memory requirement
graph.setdefault(thisEdge.sourceNode, {})
graph[thisEdge.sourceNode][thisEdge.targetNode] = thisEdge
graph.setdefault(thisEdge.targetNode, {})
graph[thisEdge.targetNode][thisEdge.sourceNode] = thisEdge
except (AWSConnectionError, Exception) as e:
import traceback, utils
utils.logError(traceback.format_exc())
raise AppError(utils.timestampStr(), "DataStore", "loadEdgeGraphForTile", e)
return graph
def _getStringList (self, thisTile):
'''
Search for lines in the file that are in Tile thisTile
return a list of those matched lines.
'''
lstLines = []
try:
fs = open ( self.filePath, 'r' )
if (self.hasHeader):
fs.readline ()
allLines = fs.readlines ()
for thisLine in allLines:
# print thisLine
cols = thisLine.split ("|")
centroidWKT= cols [8]
# centroidWKT is like "POINT (0.2343 0.2332432)"
tmpList = centroidWKT.replace ('POINT(','' ).replace ( ')','' ).split (" ")
X = float (tmpList[0] )
Y = float (tmpList[1] )
# Find the integer "floor" value of x & y co-ordinates
xVal = int (math.floor ( X ) )
yVal = int (math.floor ( Y ) )
if ( Tile ( xVal , yVal ) == thisTile ):
lstLines.append ( thisLine )
fs.close ()
except IOError as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'FileDataStore', 'open/read file', e )
return lstLines
def __getitem__(self, key):
'''
Superficially this needs to get the value matching the key in
which every sub graph is being accessed.
Matters are slightly more complicated. Consider a basic
graph
G = { nodeA: { nodeB : cost_A-B, nodeC: cost_A-C },
nodeB: { nodeA : cost_B-A },
nodeC: { nodeC : cost_C-A }
}
1. If CompositeGraph replaces G then "values" are
in fact sub-dictionaries
2. It is possible for two graphs to contain 'from' nodes
that link to the same 'to node. To handle this, it is
necessary to return a graph that includes the values of
all child graphs.
'''
lstAvailableGraphs = []
lstAvailableGraphs = self.GraphRepository.getGraphs ()
lstResults = []
for eachGraph in lstAvailableGraphs:
# create a list of (key,val) tuples
if key in eachGraph:
dictResult = eachGraph[key]
lstResults =lstResults + \
[(akey,val) for akey,val in dictResult.iteritems()]
if len (lstResults) == 0:
print "failed to find key %s " %(key)
raise AppError (utils.timestampStr (), 'DataStructures.DynamicGraph', \
'Failed to find key "%s" in CompositeGraph:' %(key), 'AppError' )
# note: the next line will flatten out any duplicates.
return dict ( lstResults ) # generate dict from key/val pairs
def __getitem__(self, key):
'''
Superficially this needs to get the value matching the key in
which every sub graph is being accessed.
Matters are slightly more complicated. Consider a basic
graph
G = { nodeA: { nodeB : cost_A-B, nodeC: cost_A-C },
nodeB: { nodeA : cost_B-A },
nodeC: { nodeC : cost_C-A }
}
1. If CompositeGraph replaces G then "values" are
in fact sub-dictionaries
2. It is possible for two graphs to contain 'from' nodes
that link to the same 'to node. To handle this, it is
necessary to return a graph that includes the values of
all child graphs.
'''
lstAvailableGraphs = []
lstAvailableGraphs = self.GraphRepository.getGraphs()
lstResults = []
for eachGraph in lstAvailableGraphs:
# create a list of (key,val) tuples
if key in eachGraph:
dictResult = eachGraph[key]
lstResults =lstResults + \
[(akey,val) for akey,val in dictResult.iteritems()]
if len(lstResults) == 0:
print "failed to find key %s " % (key)
raise AppError (utils.timestampStr (), 'DataStructures.DynamicGraph', \
'Failed to find key "%s" in CompositeGraph:' %(key), 'AppError' )
# note: the next line will flatten out any duplicates.
return dict(lstResults) # generate dict from key/val pairs
def __getitem__(self, key):
'''
Handle case in which a Tile OBJECT is used
Also increment the frequency counter for this tile.
'''
keyStr = str (key ) # key might be a Tile object
try:
val = dict.__getitem__(self, keyStr )
except KeyError as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'GraphRepository', \
'Tile with key: %s not found in GraphRepository' %(keyStr), e )
if keyStr in self.accessFrequency:
self.accessFrequency [keyStr] = self.accessFrequency [keyStr] + 1
else:
self.accessFrequency [keyStr] = 1
return val
def geocode (txtLocation):
'''
Pass txtLocation to the Nominatum geocoder API.
All being well, return a dict object with structure
{place_name = a, longitude=b, latitude = c }
'''
resultDict = None
try:
txtLocationFmt = txtLocation.replace ( ' ', '%20' )
URL = "http://nominatim.openstreetmap.org/"+\
"search?q=%s&format=xml" %(txtLocationFmt)
try:
response = urllib2.urlopen (URL)
except Exception as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'Geocoder.geocode', \
'Connecting to nomatum site for search: %s' %(txtLocation), e )
parser = make_parser()
encoding = response.headers.getparam('charset')
resultDict = {'lon':None, 'lat':None, 'display_name': None}
XMLText = response.read().decode(encoding)
response.close()
try:
parser.setContentHandler(FirstMatchParser (resultDict) )
except Exception as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'Geocoder.geocode', \
'Seting content handler for search: %s' %(txtLocation), e )
abytes = io.BytesIO(XMLText.encode('utf8'))
try:
parser.parse ( abytes )
except Exception as e:
print "here %s" %(e)
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'Geocoder.geocode', \
'Doing parse for search string : %s' %(txtLocation), e )
except Exception as e:
import traceback, utils
utils.logError ( traceback.format_exc() )
raise AppError (utils.timestampStr (), 'Geocoder.geocode', \
'General error for search : %s' %(txtLocation), e )
return resultDict
