示例#1
0
文件: db.py 项目: krf/xml-2012.3
    def reconnect(self):
        """Connect to database
        
        If the database doesn't exist, it will be created.
        \return True on success, else False (self.error indicates the error)
        """

        self._reset()

        self.session = None
        try:
            # create session
            self.session = BaseXClient.Session('localhost', 1984, 'admin', 'admin')
        except IOError as e:
            self.error = e
            return False

        try:
            self.session.execute('OPEN {0}'.format(self.databaseName))
        except IOError as e:
            self.error = e

            # attempt another try
            if "was not found" in str(e):
                success = self.createDatabase()
                if success:
                    self.reconnect()

            return False

        log.debug("Database opened: {0}".format(self.databaseName))
        return True
示例#2
0
文件: data.py 项目: krf/xml-2012.3
def readUrl(url):
    if not url.startswith('http://'):
        url = 'http://' + url

    log.debug("Reading URL: {0}".format(url))

    start= time.time()
    f = urllib2.urlopen(url)
    content = f.read()
    end = time.time()
    log.debug("  took {0} ms".format(end-start))
    return content
示例#3
0
文件: db.py 项目: krf/xml-2012.3
    def createDatabase(self):
        """Create database"""

        self._reset()

        log.debug("Trying to create the database: " + self.databaseName)

        try:
            self.session = BaseXClient.Session('localhost', 1984, 'admin', 'admin')
            self.session.execute('CREATE DB {0}'.format(self.databaseName))
        except IOError as e:
            self.error = e
            return False
示例#4
0
文件: data.py 项目: krf/xml-2012.3
def augmentTracks(db):
    """Augment tracks that are already in the database"""

    iface = TrackInterface(db)
    tracks = iface.getNonAugmentedTracks()

    log.debug("Number of non-augmented tracks: {0}".format(iface.getNonAugmentedTrackCount()))

    iterations = 0
    for track in tracks:
        tree = etree.fromstring(track)
        fileId = tree.xpath("//fileId/text()")[0]

        log.debug("Finding track details for fileId: {0} (iteration {1})".format(fileId, iterations))
        augmentOneTrack(db, fileId)
        iterations += 1

    return True
示例#5
0
文件: data.py 项目: krf/xml-2012.3
def augmentOneTrack(db, fileId):
    """Extend one specific track, by crawling gpsies.org once more

    @param fileId Track file ID
    @return True on success, False otherwise
    """

    TRACK_DETAILS_TEMPLATE = "{0}/api.do?key={1}&fileId={2}&trackDataLength=250"
    API_KEY = getApiKey()
    API_BASE_URL = options.get('Common', 'API_BASE_URL')

    def getUrl(fileId):
        url = TRACK_DETAILS_TEMPLATE.format(API_BASE_URL, API_KEY, fileId)
        return url

    url = getUrl(fileId)
    content = readUrl(url)

    tree = etree.fromstring(content)
    tracks = tree.xpath("//track") # should return exactly one track element
    assert(len(tracks) == 1)
    track = tracks[0]

    titles = tree.xpath("//track/title")
    assert(len(titles) == 1)
    title = etree.tostring(titles[0])
    if "API key required" in title:
        log.error("Title: {0}".format(title))
        raise RuntimeError("Fatal: API key broken? File ID: {0}".format(fileId))

    # transform to our database format
    success = transformTrack(track)
    if not success:
        log.debug("Failed to transform track to database format")
        return False

    # re-add node (with full details now)
    log.debug("Add track details for fileID: {0}".format(fileId))
    iface = TrackInterface(db)
    trackStr = etree.tostring(track)
    iface.addTrack(trackStr) # replaces
    return True
示例#6
0
文件: data.py 项目: krf/xml-2012.3
def crawlResultpages(db):
    """Crawl from a specific set of result pages

    Fills the data base with non-augmented tracks

    \return True if successful, else False"""

    TRACKS_LIST_TEMPLATE = "{0}/api.do?key={1}&country={2}&limit=100&resultPage={3}"
    API_KEY = getApiKey()
    API_BASE_URL = options.get('Common', 'API_BASE_URL')
    MAX_DATABASE_SIZE = options.get('Common', 'MAX_DATABASE_SIZE')

    def getUrl(resultPage):
        url = TRACKS_LIST_TEMPLATE.format(
            API_BASE_URL, API_KEY, "DE", resultPage
        )
        return url

    iface = TrackInterface(db)

    numberOfTracks = 0
    resultPage = 1
    while numberOfTracks < MAX_DATABASE_SIZE:
        url = getUrl(resultPage)
        log.debug("Parsing URL: {0}".format(url))

        content = readUrl(url)
        tree = parseContent(content)

        tracksSearch = etree.XPath("//track")
        tracks = tracksSearch(tree)
        if len(tracks) == 0:
            break

        # add tracks to database
        iface.addTracks(tracks)
        log.debug("Added {0} tracks to the database".format(len(tracks)))
        resultPage += 1

        # print out number of tracks in db
        log.debug("Tracks in database: {0}".format(iface.getTrackCount()))

    return True
示例#7
0
def main():
    """Convert from old database format"""

    db = openDatabase()
    iface = TrackInterface(db)
    tracks = db.query("//track[startPointAddress]")
    
    iterations = 0
    for trackStr in tracks:
        track = etree.fromstring(trackStr)
        fileId = TrackInterface.parseFileId(track)
        log.debug("Transforming track: {0}".format(fileId))
        success = transformTrack(track)
        if not success:
            continue

        iface.addTrack(track)
        iterations += 1

    log.debug("Finished transforming {0} tracks".format(iterations))
    log.debug("Deleting old 'tracks' path")
    db.delete("tracks")

    return 0
示例#8
0
文件: web.py 项目: krf/xml-2012.3
    StatisticsHandler, KmlHandler
import os
import sys
import tornado.ioloop
import tornado.web

settings = {
    "static_path": os.path.join(doc_root, "static"),
    "debug": True
}

application = tornado.web.Application([
    (r"/", MainHandler),
    (r"/request", RequestHandler),
    (r"/detail", DetailHandler),
    (r"/kml", KmlHandler),
    (r"/stats", StatisticsHandler)
], **settings)

if __name__ == "__main__":
    log.debug("Settings: {0}".format(settings))
    port = 8888
    application.listen(port)

    try:
        print("Starting web server on localhost:{0}".format(port))
        tornado.ioloop.IOLoop.instance().start()
    except KeyboardInterrupt:
        print("\nKeyboardInterrupt. Exit.")
    sys.exit(0)