def reconnect(self): """Connect to database If the database doesn't exist, it will be created. \return True on success, else False (self.error indicates the error) """ self._reset() self.session = None try: # create session self.session = BaseXClient.Session('localhost', 1984, 'admin', 'admin') except IOError as e: self.error = e return False try: self.session.execute('OPEN {0}'.format(self.databaseName)) except IOError as e: self.error = e # attempt another try if "was not found" in str(e): success = self.createDatabase() if success: self.reconnect() return False log.debug("Database opened: {0}".format(self.databaseName)) return True
def readUrl(url): if not url.startswith('http://'): url = 'http://' + url log.debug("Reading URL: {0}".format(url)) start= time.time() f = urllib2.urlopen(url) content = f.read() end = time.time() log.debug(" took {0} ms".format(end-start)) return content
def createDatabase(self): """Create database""" self._reset() log.debug("Trying to create the database: " + self.databaseName) try: self.session = BaseXClient.Session('localhost', 1984, 'admin', 'admin') self.session.execute('CREATE DB {0}'.format(self.databaseName)) except IOError as e: self.error = e return False
def augmentTracks(db): """Augment tracks that are already in the database""" iface = TrackInterface(db) tracks = iface.getNonAugmentedTracks() log.debug("Number of non-augmented tracks: {0}".format(iface.getNonAugmentedTrackCount())) iterations = 0 for track in tracks: tree = etree.fromstring(track) fileId = tree.xpath("//fileId/text()")[0] log.debug("Finding track details for fileId: {0} (iteration {1})".format(fileId, iterations)) augmentOneTrack(db, fileId) iterations += 1 return True
def augmentOneTrack(db, fileId): """Extend one specific track, by crawling gpsies.org once more @param fileId Track file ID @return True on success, False otherwise """ TRACK_DETAILS_TEMPLATE = "{0}/api.do?key={1}&fileId={2}&trackDataLength=250" API_KEY = getApiKey() API_BASE_URL = options.get('Common', 'API_BASE_URL') def getUrl(fileId): url = TRACK_DETAILS_TEMPLATE.format(API_BASE_URL, API_KEY, fileId) return url url = getUrl(fileId) content = readUrl(url) tree = etree.fromstring(content) tracks = tree.xpath("//track") # should return exactly one track element assert(len(tracks) == 1) track = tracks[0] titles = tree.xpath("//track/title") assert(len(titles) == 1) title = etree.tostring(titles[0]) if "API key required" in title: log.error("Title: {0}".format(title)) raise RuntimeError("Fatal: API key broken? File ID: {0}".format(fileId)) # transform to our database format success = transformTrack(track) if not success: log.debug("Failed to transform track to database format") return False # re-add node (with full details now) log.debug("Add track details for fileID: {0}".format(fileId)) iface = TrackInterface(db) trackStr = etree.tostring(track) iface.addTrack(trackStr) # replaces return True
def crawlResultpages(db): """Crawl from a specific set of result pages Fills the data base with non-augmented tracks \return True if successful, else False""" TRACKS_LIST_TEMPLATE = "{0}/api.do?key={1}&country={2}&limit=100&resultPage={3}" API_KEY = getApiKey() API_BASE_URL = options.get('Common', 'API_BASE_URL') MAX_DATABASE_SIZE = options.get('Common', 'MAX_DATABASE_SIZE') def getUrl(resultPage): url = TRACKS_LIST_TEMPLATE.format( API_BASE_URL, API_KEY, "DE", resultPage ) return url iface = TrackInterface(db) numberOfTracks = 0 resultPage = 1 while numberOfTracks < MAX_DATABASE_SIZE: url = getUrl(resultPage) log.debug("Parsing URL: {0}".format(url)) content = readUrl(url) tree = parseContent(content) tracksSearch = etree.XPath("//track") tracks = tracksSearch(tree) if len(tracks) == 0: break # add tracks to database iface.addTracks(tracks) log.debug("Added {0} tracks to the database".format(len(tracks))) resultPage += 1 # print out number of tracks in db log.debug("Tracks in database: {0}".format(iface.getTrackCount())) return True
def main(): """Convert from old database format""" db = openDatabase() iface = TrackInterface(db) tracks = db.query("//track[startPointAddress]") iterations = 0 for trackStr in tracks: track = etree.fromstring(trackStr) fileId = TrackInterface.parseFileId(track) log.debug("Transforming track: {0}".format(fileId)) success = transformTrack(track) if not success: continue iface.addTrack(track) iterations += 1 log.debug("Finished transforming {0} tracks".format(iterations)) log.debug("Deleting old 'tracks' path") db.delete("tracks") return 0
StatisticsHandler, KmlHandler import os import sys import tornado.ioloop import tornado.web settings = { "static_path": os.path.join(doc_root, "static"), "debug": True } application = tornado.web.Application([ (r"/", MainHandler), (r"/request", RequestHandler), (r"/detail", DetailHandler), (r"/kml", KmlHandler), (r"/stats", StatisticsHandler) ], **settings) if __name__ == "__main__": log.debug("Settings: {0}".format(settings)) port = 8888 application.listen(port) try: print("Starting web server on localhost:{0}".format(port)) tornado.ioloop.IOLoop.instance().start() except KeyboardInterrupt: print("\nKeyboardInterrupt. Exit.") sys.exit(0)