def fetch_data(self):
url = City.url_prefix + self._state + '/' + self._city
dest = City.file_prefix + self._abbrv + '/City/' + self._city
print 'downloading state (%s), city (%s), url (%s), dest (%s)' % (self._state, self._city, url, dest)
dst_file = File(dest)
if dst_file.exists() is True:
print '........Data for %s, %s already present' % (self._state, self._city)
return
download = Download(url, dest)
download.download_cookie()
def fetch_data(self):
url = City.url_prefix + self._state + '/' + self._city
dest = City.file_prefix + self._abbrv + '/City/' + self._city
print 'downloading state (%s), city (%s), url (%s), dest (%s)' % (
self._state, self._city, url, dest)
dst_file = File(dest)
if dst_file.exists() is True:
print '........Data for %s, %s already present' % (self._state,
self._city)
return
download = Download(url, dest)
download.download_cookie()
def fetch_cities_in_state(self):
for alpha in self._alpha:
url = State.url_prefix + self._abbrv + "/" + alpha
dest = State.file_prefix + self._abbrv + "/" + alpha
print "downloading state (%s), url (%s), state (%s)" % (self._state, url, dest)
dir = Dir(dest)
if dir.exists() is False:
dir.create_if_needed()
# check if data is present
data_file = File(dest + "/file")
if data_file.exists() is True:
print "data present for state %s, %s" % (self._state, alpha)
continue
download = Download(url, dest + "/file")
download.download()
def fetch_cities_in_state(self):
for alpha in self._alpha:
url = State.url_prefix + self._abbrv + '/' + alpha
dest = State.file_prefix + self._abbrv + '/' + alpha
print 'downloading state (%s), url (%s), state (%s)' % (
self._state, url, dest)
dir = Dir(dest)
if dir.exists() is False:
dir.create_if_needed()
# check if data is present
data_file = File(dest + '/file')
if data_file.exists() is True:
print 'data present for state %s, %s' % (self._state, alpha)
continue
download = Download(url, dest + '/file')
download.download()
def fetch_cities_data(self):
dest = State.file_prefix + self._abbrv + "/City"
dir = Dir(dest)
if dir.exists() is False:
dir.create_if_needed()
for alpha in self._alpha:
data = File(State.file_prefix + self._abbrv + "/" + alpha + "/file")
if data.exists() is True:
# parse and get a list of cities
print "parsing city data for (%s, %s)" % (self._state, alpha)
cities = CityParser(self._abbrv, data)
for c in cities.parse():
this_city = City(self._abbrv, State.state_map[self._abbrv], c)
try:
this_city.fetch_data()
except Exception:
print "error data for (%s, %s)" % (self._state, alpha)
else:
print "...... no data for %s, %s" % (self._abbrv, alpha)
def fetch_cities_data(self):
dest = State.file_prefix + self._abbrv + '/City'
dir = Dir(dest)
if dir.exists() is False:
dir.create_if_needed()
for alpha in self._alpha:
data = File(State.file_prefix + self._abbrv + '/' + alpha +
'/file')
if data.exists() is True:
# parse and get a list of cities
print 'parsing city data for (%s, %s)' % (self._state, alpha)
cities = CityParser(self._abbrv, data)
for c in cities.parse():
this_city = City(self._abbrv, State.state_map[self._abbrv],
c)
try:
this_city.fetch_data()
except Exception:
print 'error data for (%s, %s)' % (self._state, alpha)
else:
print '...... no data for %s, %s' % (self._abbrv, alpha)
def extract_and_publish(self, csvf):
# parse the data out here
dest = City.file_prefix + self._abbrv + '/City/' + self._city
dst_file = File(dest)
self._rating = CitySchoolRatingParser(dst_file).parse()
csvf.writerow([self._state, self._city, self._rating])