def _getStringList(self, thisTile): conn = self._getS3Connection() bucketRef = None keyID = None try: bucketRef = conn.get_bucket(self.bucketPrefix) except (AWSConnectionError, Exception) as e: import traceback, utils utils.logError(traceback.format_exc()) errStr = "Opening key %s" % (keyID) raise AppError(utils.timestampStr(), "DataStore", errStr, e) strEdges = None tileID = thisTile.getID() keyRef = None try: keyRef = bucketRef.get_key(thisTile.getID()) strEdges = keyRef.get_contents_as_string() except (AWSConnectionError, Exception) as e: import traceback, utils utils.logError(traceback.format_exc()) errStr = "Reading bucket key: %s" % (tileID) raise AppError(utils.timestampStr(), "DataStore", errStr, e) resultList = [] someLines = strEdges.split("\n") listLen = len(someLines) count = 0 while len(someLines) > 0: aLine = someLines.pop() if len(aLine) > 0: resultList.append(aLine) conn.close() return resultList
def _getStringList (self, thisTile): conn = self._getS3Connection () bucketRef = None keyID = None try: bucketRef =conn.get_bucket ( self.bucketPrefix ); except (AWSConnectionError, Exception) as e: import traceback, utils utils.logError ( traceback.format_exc() ) errStr = 'Opening key %s' %( keyID ) raise AppError (utils.timestampStr (), 'DataStore', errStr, e ) strEdges = None tileID = thisTile.getID () keyRef = None try: keyRef = bucketRef.get_key ( thisTile.getID ()) strEdges = keyRef.get_contents_as_string() except (AWSConnectionError, Exception) as e: import traceback, utils utils.logError ( traceback.format_exc() ) errStr = 'Reading bucket key: %s' %( tileID ) raise AppError (utils.timestampStr (), 'DataStore', errStr, e ) resultList = [] someLines = strEdges.split ('\n') listLen = len ( someLines ) count = 0 while (len ( someLines ) > 0 ): aLine = someLines.pop () if len (aLine ) > 0: resultList.append ( aLine ) conn.close () return resultList
def _getS3Connection (self): if 'AWS_ACCESS_KEY_ID' in os.environ: AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] else : errStr ="AWS_ACCESS_KEY_ID not set" raise AppError (utils.timestampStr (), 'DataStore', errStr, '' ) if 'AWS_SECRET_ACCESS_KEY' in os.environ: AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] else : errStr ="AWS_SECRET_ACCESS_KEY not set" raise AppError (utils.timestampStr (), 'DataStore', errStr, '' ) try: conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) except (AWSConnectionError, Exception) as e: import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'DataStore', 'creating AWS connection', e ) else: return conn
def _getS3Connection(self): if "AWS_ACCESS_KEY_ID" in os.environ: AWS_ACCESS_KEY_ID = os.environ["AWS_ACCESS_KEY_ID"] else: errStr = "AWS_ACCESS_KEY_ID not set" raise AppError(utils.timestampStr(), "DataStore", errStr, "") if "AWS_SECRET_ACCESS_KEY" in os.environ: AWS_SECRET_ACCESS_KEY = os.environ["AWS_SECRET_ACCESS_KEY"] else: errStr = "AWS_SECRET_ACCESS_KEY not set" raise AppError(utils.timestampStr(), "DataStore", errStr, "") try: conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) except (AWSConnectionError, Exception) as e: import traceback, utils utils.logError(traceback.format_exc()) raise AppError(utils.timestampStr(), "DataStore", "creating AWS connection", e) else: return conn
def getKeys (self): ''' Return a list of all of the Tile (key)s currently held in the repository ''' lst= [] try: for key,val in self.iteritems (): lst.append ( key ) except Exception as e: import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'GraphRepository', \ 'Iterating repository, at key %s:' %(key), e ) return lst
def loadEdgeGraphForTile (self, thisTile): ''' A graph G is defined so: G = { nodeA: { nodeB : cost_A-B, nodeC: cost_A-C }, nodeB: { nodeA : cost_B-A }, nodeC: { nodeC : cost_C-A } } where cost_X_Y is in an instance of DataStructures.Edge The graph created here will be a subclass of: - dict - gis.Tile ''' try: strList = self._getStringList ( thisTile ) graph = {} for thisLine in strList: thisEdge = self._createEdgeFromLine (thisLine) # note, do not implement reverse costs/Edges at this # stage, to reduce memory requirement graph.setdefault ( thisEdge.sourceNode, {} ) graph[thisEdge.sourceNode][thisEdge.targetNode] = thisEdge graph.setdefault ( thisEdge.targetNode, {} ) graph[thisEdge.targetNode][thisEdge.sourceNode] = thisEdge except (AWSConnectionError, Exception) as e: import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'DataStore', \ 'loadEdgeGraphForTile', e ) return graph
def _getStringList(self, thisTile): """ Search for lines in the file that are in Tile thisTile return a list of those matched lines. """ lstLines = [] try: fs = open(self.filePath, "r") if self.hasHeader: fs.readline() allLines = fs.readlines() for thisLine in allLines: # print thisLine cols = thisLine.split("|") centroidWKT = cols[8] # centroidWKT is like "POINT (0.2343 0.2332432)" tmpList = centroidWKT.replace("POINT(", "").replace(")", "").split(" ") X = float(tmpList[0]) Y = float(tmpList[1]) # Find the integer "floor" value of x & y co-ordinates xVal = int(math.floor(X)) yVal = int(math.floor(Y)) if Tile(xVal, yVal) == thisTile: lstLines.append(thisLine) fs.close() except IOError as e: import traceback, utils utils.logError(traceback.format_exc()) raise AppError(utils.timestampStr(), "FileDataStore", "open/read file", e) return lstLines
def loadEdgeGraphForTile(self, thisTile): """ A graph G is defined so: G = { nodeA: { nodeB : cost_A-B, nodeC: cost_A-C }, nodeB: { nodeA : cost_B-A }, nodeC: { nodeC : cost_C-A } } where cost_X_Y is in an instance of DataStructures.Edge The graph created here will be a subclass of: - dict - gis.Tile """ try: strList = self._getStringList(thisTile) graph = {} for thisLine in strList: thisEdge = self._createEdgeFromLine(thisLine) # note, do not implement reverse costs/Edges at this # stage, to reduce memory requirement graph.setdefault(thisEdge.sourceNode, {}) graph[thisEdge.sourceNode][thisEdge.targetNode] = thisEdge graph.setdefault(thisEdge.targetNode, {}) graph[thisEdge.targetNode][thisEdge.sourceNode] = thisEdge except (AWSConnectionError, Exception) as e: import traceback, utils utils.logError(traceback.format_exc()) raise AppError(utils.timestampStr(), "DataStore", "loadEdgeGraphForTile", e) return graph
def _getStringList (self, thisTile): ''' Search for lines in the file that are in Tile thisTile return a list of those matched lines. ''' lstLines = [] try: fs = open ( self.filePath, 'r' ) if (self.hasHeader): fs.readline () allLines = fs.readlines () for thisLine in allLines: # print thisLine cols = thisLine.split ("|") centroidWKT= cols [8] # centroidWKT is like "POINT (0.2343 0.2332432)" tmpList = centroidWKT.replace ('POINT(','' ).replace ( ')','' ).split (" ") X = float (tmpList[0] ) Y = float (tmpList[1] ) # Find the integer "floor" value of x & y co-ordinates xVal = int (math.floor ( X ) ) yVal = int (math.floor ( Y ) ) if ( Tile ( xVal , yVal ) == thisTile ): lstLines.append ( thisLine ) fs.close () except IOError as e: import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'FileDataStore', 'open/read file', e ) return lstLines
def __getitem__(self, key): ''' Superficially this needs to get the value matching the key in which every sub graph is being accessed. Matters are slightly more complicated. Consider a basic graph G = { nodeA: { nodeB : cost_A-B, nodeC: cost_A-C }, nodeB: { nodeA : cost_B-A }, nodeC: { nodeC : cost_C-A } } 1. If CompositeGraph replaces G then "values" are in fact sub-dictionaries 2. It is possible for two graphs to contain 'from' nodes that link to the same 'to node. To handle this, it is necessary to return a graph that includes the values of all child graphs. ''' lstAvailableGraphs = [] lstAvailableGraphs = self.GraphRepository.getGraphs () lstResults = [] for eachGraph in lstAvailableGraphs: # create a list of (key,val) tuples if key in eachGraph: dictResult = eachGraph[key] lstResults =lstResults + \ [(akey,val) for akey,val in dictResult.iteritems()] if len (lstResults) == 0: print "failed to find key %s " %(key) raise AppError (utils.timestampStr (), 'DataStructures.DynamicGraph', \ 'Failed to find key "%s" in CompositeGraph:' %(key), 'AppError' ) # note: the next line will flatten out any duplicates. return dict ( lstResults ) # generate dict from key/val pairs
def __getitem__(self, key): ''' Superficially this needs to get the value matching the key in which every sub graph is being accessed. Matters are slightly more complicated. Consider a basic graph G = { nodeA: { nodeB : cost_A-B, nodeC: cost_A-C }, nodeB: { nodeA : cost_B-A }, nodeC: { nodeC : cost_C-A } } 1. If CompositeGraph replaces G then "values" are in fact sub-dictionaries 2. It is possible for two graphs to contain 'from' nodes that link to the same 'to node. To handle this, it is necessary to return a graph that includes the values of all child graphs. ''' lstAvailableGraphs = [] lstAvailableGraphs = self.GraphRepository.getGraphs() lstResults = [] for eachGraph in lstAvailableGraphs: # create a list of (key,val) tuples if key in eachGraph: dictResult = eachGraph[key] lstResults =lstResults + \ [(akey,val) for akey,val in dictResult.iteritems()] if len(lstResults) == 0: print "failed to find key %s " % (key) raise AppError (utils.timestampStr (), 'DataStructures.DynamicGraph', \ 'Failed to find key "%s" in CompositeGraph:' %(key), 'AppError' ) # note: the next line will flatten out any duplicates. return dict(lstResults) # generate dict from key/val pairs
def __getitem__(self, key): ''' Handle case in which a Tile OBJECT is used Also increment the frequency counter for this tile. ''' keyStr = str (key ) # key might be a Tile object try: val = dict.__getitem__(self, keyStr ) except KeyError as e: import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'GraphRepository', \ 'Tile with key: %s not found in GraphRepository' %(keyStr), e ) if keyStr in self.accessFrequency: self.accessFrequency [keyStr] = self.accessFrequency [keyStr] + 1 else: self.accessFrequency [keyStr] = 1 return val
def geocode (txtLocation): ''' Pass txtLocation to the Nominatum geocoder API. All being well, return a dict object with structure {place_name = a, longitude=b, latitude = c } ''' resultDict = None try: txtLocationFmt = txtLocation.replace ( ' ', '%20' ) URL = "http://nominatim.openstreetmap.org/"+\ "search?q=%s&format=xml" %(txtLocationFmt) try: response = urllib2.urlopen (URL) except Exception as e: import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'Geocoder.geocode', \ 'Connecting to nomatum site for search: %s' %(txtLocation), e ) parser = make_parser() encoding = response.headers.getparam('charset') resultDict = {'lon':None, 'lat':None, 'display_name': None} XMLText = response.read().decode(encoding) response.close() try: parser.setContentHandler(FirstMatchParser (resultDict) ) except Exception as e: import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'Geocoder.geocode', \ 'Seting content handler for search: %s' %(txtLocation), e ) abytes = io.BytesIO(XMLText.encode('utf8')) try: parser.parse ( abytes ) except Exception as e: print "here %s" %(e) import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'Geocoder.geocode', \ 'Doing parse for search string : %s' %(txtLocation), e ) except Exception as e: import traceback, utils utils.logError ( traceback.format_exc() ) raise AppError (utils.timestampStr (), 'Geocoder.geocode', \ 'General error for search : %s' %(txtLocation), e ) return resultDict