Пример #1
0
    def fetch_data(self):

        url = City.url_prefix + self._state + '/' + self._city
        dest = City.file_prefix + self._abbrv + '/City/' + self._city
        print 'downloading state (%s), city (%s), url (%s), dest (%s)' % (self._state, self._city, url, dest)

        dst_file = File(dest)
        if dst_file.exists() is True:
            print '........Data for %s, %s already present' % (self._state, self._city)
            return

        download = Download(url, dest)
        download.download_cookie()
Пример #2
0
    def fetch_data(self):

        url = City.url_prefix + self._state + '/' + self._city
        dest = City.file_prefix + self._abbrv + '/City/' + self._city
        print 'downloading state (%s), city (%s), url (%s), dest (%s)' % (
            self._state, self._city, url, dest)

        dst_file = File(dest)
        if dst_file.exists() is True:
            print '........Data for %s, %s already present' % (self._state,
                                                               self._city)
            return

        download = Download(url, dest)
        download.download_cookie()
Пример #3
0
    def fetch_cities_in_state(self):
        for alpha in self._alpha:
            url = State.url_prefix + self._abbrv + "/" + alpha
            dest = State.file_prefix + self._abbrv + "/" + alpha
            print "downloading state (%s), url (%s), state (%s)" % (self._state, url, dest)
            dir = Dir(dest)
            if dir.exists() is False:
                dir.create_if_needed()

            # check if data is present
            data_file = File(dest + "/file")
            if data_file.exists() is True:
                print "data present for state %s, %s" % (self._state, alpha)
                continue

            download = Download(url, dest + "/file")
            download.download()
Пример #4
0
    def fetch_cities_in_state(self):
        for alpha in self._alpha:
            url = State.url_prefix + self._abbrv + '/' + alpha
            dest = State.file_prefix + self._abbrv + '/' + alpha
            print 'downloading state (%s), url (%s), state (%s)' % (
                self._state, url, dest)
            dir = Dir(dest)
            if dir.exists() is False:
                dir.create_if_needed()

            # check if data is present
            data_file = File(dest + '/file')
            if data_file.exists() is True:
                print 'data present for state %s, %s' % (self._state, alpha)
                continue

            download = Download(url, dest + '/file')
            download.download()
Пример #5
0
    def fetch_cities_data(self):
        dest = State.file_prefix + self._abbrv + "/City"
        dir = Dir(dest)
        if dir.exists() is False:
            dir.create_if_needed()

        for alpha in self._alpha:
            data = File(State.file_prefix + self._abbrv + "/" + alpha + "/file")
            if data.exists() is True:
                # parse and get a list of cities
                print "parsing city data for (%s, %s)" % (self._state, alpha)
                cities = CityParser(self._abbrv, data)
                for c in cities.parse():
                    this_city = City(self._abbrv, State.state_map[self._abbrv], c)
                    try:
                        this_city.fetch_data()
                    except Exception:
                        print "error data for (%s, %s)" % (self._state, alpha)
            else:
                print "...... no data for %s, %s" % (self._abbrv, alpha)
Пример #6
0
    def fetch_cities_data(self):
        dest = State.file_prefix + self._abbrv + '/City'
        dir = Dir(dest)
        if dir.exists() is False:
            dir.create_if_needed()

        for alpha in self._alpha:
            data = File(State.file_prefix + self._abbrv + '/' + alpha +
                        '/file')
            if data.exists() is True:
                # parse and get a list of cities
                print 'parsing city data for (%s, %s)' % (self._state, alpha)
                cities = CityParser(self._abbrv, data)
                for c in cities.parse():
                    this_city = City(self._abbrv, State.state_map[self._abbrv],
                                     c)
                    try:
                        this_city.fetch_data()
                    except Exception:
                        print 'error data for (%s, %s)' % (self._state, alpha)
            else:
                print '...... no data for %s, %s' % (self._abbrv, alpha)
Пример #7
0
 def extract_and_publish(self, csvf):
     # parse the data out here
     dest = City.file_prefix + self._abbrv + '/City/' + self._city
     dst_file = File(dest)
     self._rating = CitySchoolRatingParser(dst_file).parse()
     csvf.writerow([self._state, self._city, self._rating])